{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Reformatting Ubuntu Dialogue Corpus for Chatbot Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Description"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**How to run this notebook**: \n",
    "1. Download the data from the [github repository](https://github.com/rkadlec/ubuntu-ranking-dataset-creator)\n",
    "2. CD to the downloaded directory and run ```./generate.sh -t -l```. I downloaded 1 mill. train samples with p = 1.0. \n",
    "3. Change the value of DATA_DIR below to your project path root.\n",
    "\n",
    "I chose to copy the first 10k lines of data into a \"sample\" directory so I could quickly play around with the data, and then use the full dataset when done.\n",
    "\n",
    "Random facts [from paper](https://arxiv.org/abs/1506.08909):\n",
    "* 2-way (dyadic) conversation, as opposed to multi-participant.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os.path\n",
    "import pdb\n",
    "import pandas as pd\n",
    "from pprint import pprint\n",
    "\n",
    "#DATA_DIR = '/home/brandon/terabyte/Datasets/ubuntu_dialogue_corpus/'\n",
    "DATA_DIR = '/home/brandon/ubuntu_dialogue_corpus/src/' # sample/'\n",
    "TRAIN_PATH = DATA_DIR + 'train.csv'\n",
    "VALID_PATH = DATA_DIR + 'valid.csv'\n",
    "TEST_PATH = DATA_DIR + 'test.csv'\n",
    "\n",
    "def get_training():\n",
    "    \"\"\"Returns dataframe data from train.csv \"\"\"\n",
    "    # First, we need to load the data directly into a dataframe from the train.csv file. \n",
    "    df_train = pd.read_csv(TRAIN_PATH)\n",
    "    # Remove all examples with label = 0. (why would i want to train on false examples?)\n",
    "    df_train = df_train.loc[df_train['Label'] == 1.0]\n",
    "    # Don't care about the pandas indices in the df, so remove them.\n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_train = df_train[df_train.columns[:2]]\n",
    "    return df_train\n",
    "\n",
    "def get_validation():\n",
    "    \"\"\"Returns data from valid.csv \"\"\"\n",
    "    # First, we need to load the data directly into a dataframe from the train.csv file. \n",
    "    df_valid = pd.read_csv(VALID_PATH)\n",
    "    first_two_cols = df_valid.columns[:2]\n",
    "    df_valid = df_valid[first_two_cols]\n",
    "    df_valid.columns = ['Context', 'Utterance']\n",
    "    return df_valid\n",
    "\n",
    "df_train = get_training()\n",
    "df_valid = get_validation()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Functions for Visualization and Reformatting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now get all of the data in a single string and make a 'vocabulary' (unique words). \n",
    "import nltk, re, pprint\n",
    "from nltk import word_tokenize\n",
    "import pdb\n",
    "\n",
    "def print_single_turn(turn: str):\n",
    "    as_list_of_utters = turn.split('__eou__')[:-1]\n",
    "    for idx_utter, utter in enumerate(as_list_of_utters):\n",
    "        print(\"\\t>>>\", utter)\n",
    "\n",
    "def print_conversation(df, index=0):\n",
    "    \"\"\"Display the ith conversation in nice format.\"\"\"\n",
    "    \n",
    "    # Get the row identified by 'index'. \n",
    "    context_entry = df['Context'].values[index]\n",
    "    target        = df['Utterance'].values[index]\n",
    "    \n",
    "    # Split returns a blank last entry, so don't store.\n",
    "    turns = context_entry.split('__eot__')[:-1]\n",
    "    print('--------------------- CONTEXT ------------------- ')\n",
    "    for idx_turn, turn in enumerate(turns):\n",
    "        print(\"\\nUser {}: \".format(idx_turn % 2))\n",
    "        print_single_turn(turn)\n",
    "    print('\\n--------------------- RESPONSE ------------------- ')\n",
    "    print(\"\\nUser {}: \".format(len(turns) % 2))\n",
    "    print_single_turn(target)\n",
    "        \n",
    "def get_user_arrays(df):\n",
    "    \"\"\"Returns two arrays of every other turn. \n",
    "    Specifically:\n",
    "        len(returned array) is number of rows in df.  I SURE HOPE NOT!\n",
    "        each entry is a numpy array. \n",
    "        each numpy array contains utterances as entries. \n",
    "    \"\"\"\n",
    "    userOne = []\n",
    "    userTwo = []\n",
    "    contexts = df['Context'].values\n",
    "    targets  = df['Utterance'].values\n",
    "    assert(len(contexts) == len(targets))\n",
    "    \n",
    "    for i in range(len(contexts)):\n",
    "        # combined SINGLE CONVERSATION ENTRY of multiple turns each with multiple utterances.\n",
    "        list_of_turns = contexts[i].lower().split('__eot__')[:-1] + [targets[i].lower()]\n",
    "        # make sure even number of entries\n",
    "        if len(list_of_turns) % 2 != 0:\n",
    "            list_of_turns = list_of_turns[:-1]\n",
    "        # strip out the __eou__ occurences (leading space bc otherwise would result in two spaces)\n",
    "        new_list_of_turns = []\n",
    "        for turn in list_of_turns:\n",
    "            utter_list = turn.lower().split(\" __eou__\")\n",
    "            #if len(utter_list) > 3:\n",
    "            #   utter_list = utter_list[:3]\n",
    "            new_list_of_turns.append(\"\".join(utter_list))\n",
    "        #list_of_turns = [re.sub(' __eou__', '', t) for t in list_of_turns]\n",
    "        userOneThisConvo = new_list_of_turns[0::2]\n",
    "        userTwoThisConvo = new_list_of_turns[1::2]\n",
    "        userOne += userOneThisConvo \n",
    "        userTwo += userTwoThisConvo\n",
    "    assert(len(userOne) == len(userTwo))\n",
    "    return userOne, userTwo\n",
    "\n",
    "def save_to_file(fname, arr):\n",
    "    with open(DATA_DIR+fname,\"w\") as f:\n",
    "        for line in arr:\n",
    "            f.write(line + \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### At a Glance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Context</th>\n",
       "      <th>Utterance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000000</td>\n",
       "      <td>1000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>957096</td>\n",
       "      <td>849957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>! ops __eou__ __eot__ ? __eou__ __eot__</td>\n",
       "      <td>thank __eou__</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>14</td>\n",
       "      <td>11658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        Context      Utterance\n",
       "count                                   1000000        1000000\n",
       "unique                                   957096         849957\n",
       "top     ! ops __eou__ __eot__ ? __eou__ __eot__  thank __eou__\n",
       "freq                                         14          11658"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Context</th>\n",
       "      <th>Utterance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i think we could import the old comment via rsync , but from there we need to go via email . I think it be easier than cache the status on each bug and than import bits here and there __eou__ __eot__ it would be very easy to keep a hash db of message-ids __eou__ sound good __eou__ __eot__ ok __eou__ perhaps we can ship an ad-hoc apt_prefereces __eou__ __eot__ version ? __eou__ __eot__ thank __eou__ __eot__ not yet __eou__ it be cover by your insurance ? __eou__ __eot__ yes __eou__ but it 's ...</td>\n",
       "      <td>basically each xfree86 upload will NOT force users to upgrade 100Mb of fonts for nothing __eou__ no something i do in my spare time . __eou__</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I 'm not suggest all - only the ones you modify . __eou__ __eot__ ok , it sound like you 're agree with me , then __eou__ though rather than `` the ones we modify '' , my idea be `` the ones we need to merge '' __eou__ __eot__</td>\n",
       "      <td>oh ? oops . __eou__</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               Context  \\\n",
       "0  i think we could import the old comment via rsync , but from there we need to go via email . I think it be easier than cache the status on each bug and than import bits here and there __eou__ __eot__ it would be very easy to keep a hash db of message-ids __eou__ sound good __eou__ __eot__ ok __eou__ perhaps we can ship an ad-hoc apt_prefereces __eou__ __eot__ version ? __eou__ __eot__ thank __eou__ __eot__ not yet __eou__ it be cover by your insurance ? __eou__ __eot__ yes __eou__ but it 's ...   \n",
       "1                                                                                                                                                                                                                                                                                   I 'm not suggest all - only the ones you modify . __eou__ __eot__ ok , it sound like you 're agree with me , then __eou__ though rather than `` the ones we modify '' , my idea be `` the ones we need to merge '' __eou__ __eot__   \n",
       "\n",
       "                                                                                                                                       Utterance  \n",
       "0  basically each xfree86 upload will NOT force users to upgrade 100Mb of fonts for nothing __eou__ no something i do in my spare time . __eou__  \n",
       "1                                                                                                                            oh ? oops . __eou__  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.options.display.max_colwidth = 500\n",
    "df_train.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------- CONTEXT ------------------- \n",
      "\n",
      "User 0: \n",
      "\t>>> interest \n",
      "\t>>>  grub-install work with / be ext3 , fail when it be xfs \n",
      "\t>>>  i think d-i instal the relevant kernel for your machine . i have a p4 and its instal the 386 kernel \n",
      "\t>>>  holy crap a lot of stuff get instal by default : ) \n",
      "\t>>>  YOU ARE INSTALLING VIM ON A BOX OF MINE \n",
      "\t>>>  ; ) \n",
      "\n",
      "User 1: \n",
      "\t>>>  more like osx than debian ; ) \n",
      "\t>>>  we have a selection of python modules available for great justice ( and python development ) \n",
      "\n",
      "User 0: \n",
      "\t>>>  2.8 be fix them iirc \n",
      "\n",
      "User 1: \n",
      "\t>>>  pong \n",
      "\t>>>  vino will be in \n",
      "\t>>>  enjoy ubuntu ? \n",
      "\n",
      "User 0: \n",
      "\t>>>  tell me to come here \n",
      "\t>>>  suggest thursday as a good day to come \n",
      "\n",
      "User 1: \n",
      "\t>>>  we freeze versions a while back : ) \n",
      "\t>>>  you come today or thursday ? \n",
      "\t>>>  we 're consider shift it \n",
      "\t>>>  yay \n",
      "\t>>>  enjoy ubuntu ? \n",
      "\t>>>  usplash ! \n",
      "\n",
      "User 0: \n",
      "\t>>>  thats the one \n",
      "\n",
      "User 1: \n",
      "\t>>>  so i saw your email with the mockup at the airport , but it have n't appear now that i 've pull my mail : | \n",
      "\n",
      "User 0: \n",
      "\t>>>  i 've get a better one now too , give me a minute \n",
      "\t>>>  we 've get rh9 instal on most desktops . you want me to look at up2date , right ? \n",
      "\n",
      "User 1: \n",
      "\t>>>  aha ! no , the gui thingy \n",
      "\t>>>  it 's more wizardy \n",
      "\t>>>  so the first page be okayish \n",
      "\t>>>  we can do a whole load better on the second page ( icons , translate descriptions ) \n",
      "\t>>>  but that 's the kind of thing i be think about \n",
      "\t>>>  ( a single big treeview would get very scary , very quickly ) \n",
      "\t>>>  sure it 's not a hurricane ? \n",
      "\n",
      "User 0: \n",
      "\t>>>  i think experimental be get 2.8 too \n",
      "\t>>>  let him work on # 1217 : ) \n",
      "\n",
      "User 1: \n",
      "\t>>>  we call it 'universe ' ; ) \n",
      "\t>>>  haha \n",
      "\t>>>  ooh , totally \n",
      "\n",
      "User 0: \n",
      "\t>>>  i want it on in sarge too but nobody else agree \n",
      "\n",
      "--------------------- RESPONSE ------------------- \n",
      "\n",
      "User 1: \n",
      "\t>>> i fully endorse this suggestion < /quimby > \n",
      "\t>>>  how do your reinstall go ? \n"
     ]
    }
   ],
   "source": [
    "print_conversation(df_train, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Turn-Based DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserOne</th>\n",
       "      <th>UserTwo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i think we could import the old comment via rs...</td>\n",
       "      <td>it would be very easy to keep a hash db of me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ok perhaps we can ship an ad-hoc apt_prefereces</td>\n",
       "      <td>version ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>thank</td>\n",
       "      <td>not yet it be cover by your insurance ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>yes but it 's really not the right time : / w...</td>\n",
       "      <td>you will be move into your house soon ? post ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>how urgent be # 896 ?</td>\n",
       "      <td>not particularly urgent , but a policy violat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>i agree that we should kill the -novtswitch</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>would you consider a package split a feature ?</td>\n",
       "      <td>context ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>split xfonts* out of xfree86* . one upload fo...</td>\n",
       "      <td>split the source package you mean ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>yes . same binary package .</td>\n",
       "      <td>i would prefer to avoid it at this stage . th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>i 'm not suggest all - only the ones you modif...</td>\n",
       "      <td>ok , it sound like you 're agree with me , th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>afternoon all not entirely relate to warty , b...</td>\n",
       "      <td>here</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>you might want to know that thinice in warty ...</td>\n",
       "      <td>and apparently gnome be suddently almost perf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>can i file the panel not link to eds ? : )</td>\n",
       "      <td>be you use alt ? or the windows key ? wait fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>i just restart x and now nautilus wo n't show...</td>\n",
       "      <td>do you think we have any interest to have hal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>be it a know bug that g-s-t do n't know what ...</td>\n",
       "      <td>somebody should really kick that guy *hard* i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>arse . xt-dev ? i add libx11-dev so just libx...</td>\n",
       "      <td>we have plan to speak about menu organisation...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>be away , you say ? nope</td>\n",
       "      <td>the warty repository ok , fine . thanks nice ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>you 'll be glad to know i 've fix my miss arr...</td>\n",
       "      <td>i 've upload the gnome-vfs without hal suppor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>should g2 in ubuntu do the magic dont-focus-w...</td>\n",
       "      <td>we 'll have a bof about this so you 're come t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>interest grub-install work with / be ext3 , fa...</td>\n",
       "      <td>more like osx than debian ; ) we have a selec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2.8 be fix them iirc</td>\n",
       "      <td>pong vino will be in enjoy ubuntu ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>tell me to come here suggest thursday as a go...</td>\n",
       "      <td>we freeze versions a while back : ) you come ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>thats the one</td>\n",
       "      <td>so i saw your email with the mockup at the ai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>i 've get a better one now too , give me a mi...</td>\n",
       "      <td>aha ! no , the gui thingy it 's more wizardy ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>i think experimental be get 2.8 too let him w...</td>\n",
       "      <td>we call it 'universe ' ; ) haha ooh , totally</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>i want it on in sarge too but nobody else agree</td>\n",
       "      <td>i fully endorse this suggestion &lt; /quimby &gt; ho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>and because python give mark a woody</td>\n",
       "      <td>i 'm not sure if we 're mean to talk about th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>and i think we be a `` pant off '' kind of co...</td>\n",
       "      <td>mono 1.0 ? dude , that 's go to be a barrel o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>there be an accompany irc conversation to tha...</td>\n",
       "      <td>but debian/ be also part of diff.gz ... you c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>notwarty , hth , hand , kthxbye &lt; g &gt; everyon...</td>\n",
       "      <td>which ? the best feature of the new imac be t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>this mornings run fine .</td>\n",
       "      <td>yep , it be . could you add hurd-i386 amd64 t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>do - i assume that you 'll poke elmo to fresh...</td>\n",
       "      <td>do n't know i have to i 'll remind him next ti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>know bug . check bugs.debian.org/src : xfree86...</td>\n",
       "      <td>i 'm wait approval to join the sounder list ....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>setuid /bin/mount be go to be disable in favou...</td>\n",
       "      <td>under all circumstances , i take it ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>i would say so</td>\n",
       "      <td>do</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>i will accept responsibility for the addition...</td>\n",
       "      <td>indeed . it 's a recommends : thing , fix act...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>we need to make sure that cdroms ( and other r...</td>\n",
       "      <td>presumably i should make the same quietinit- ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>yes</td>\n",
       "      <td>do ( be the actual option change in sysvinit ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>sysvinit usplash will also check for it and n...</td>\n",
       "      <td>the patch as post be wrong , but i 've test w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>if it work , can you patch it and upload it ?</td>\n",
       "      <td>sure , let 's make it tomorrow though : - )</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>np</td>\n",
       "      <td>hm , that eject change be go to need some tes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>pmount just wrap mount , so long as you be a....</td>\n",
       "      <td>that 's not my point at all</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>i 'll double check that thats the whole reaso...</td>\n",
       "      <td>maybe just fall back to something like the ol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>thats a little difficult with the way it be c...</td>\n",
       "      <td>do n't have to be exact by any mean be it imp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>what do the fb corruption look like ?</td>\n",
       "      <td>lot of dot and squggles , text be partly read...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>which package ?</td>\n",
       "      <td>^^^</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>they build 'em sturdy down there , apparently...</td>\n",
       "      <td>over here they install fine , but then apt wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>try apt-get clean ?</td>\n",
       "      <td>that fix it , thank</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>that issue should n't affect upgrade from woo...</td>\n",
       "      <td>pitti : i 'm in group plugdev , it 's still n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>one difference be that some pcs have a line o...</td>\n",
       "      <td>removable devices have be bad for me , but it...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>i think we nail the issue ( executable permis...</td>\n",
       "      <td>it 's copy into their cache already , be n't ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>hmm , good point nono , ( b ) give the cd to ...</td>\n",
       "      <td>if we be use grub only , should we turn off do...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>poke</td>\n",
       "      <td>try to remember if i ask if you be ppc or i38...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>yeah , series 1 h8 that debconf should be con...</td>\n",
       "      <td>what kind of puppies ? awesome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>you ping the other day ? ( i only just get to ...</td>\n",
       "      <td>i ping a few minutes ago do you make a usplas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>right click on the desktop be not work for me</td>\n",
       "      <td>nautilus manage the desktop ? the icons be di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>anybody else see fb corruption during boot ? g...</td>\n",
       "      <td>which video card ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>no , it 's my home desktop</td>\n",
       "      <td>i remember fb break on that laptop and iirc it...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>that typically mean that apt be see multiple p...</td>\n",
       "      <td>there 's something definitely screwy with the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>stock warty apt-get install `debootstrap -- p...</td>\n",
       "      <td>what kernel be you run ? # 268154 , yeah , re...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               UserOne  \\\n",
       "0    i think we could import the old comment via rs...   \n",
       "1     ok perhaps we can ship an ad-hoc apt_prefereces    \n",
       "2                                               thank    \n",
       "3     yes but it 's really not the right time : / w...   \n",
       "4                               how urgent be # 896 ?    \n",
       "5         i agree that we should kill the -novtswitch    \n",
       "6      would you consider a package split a feature ?    \n",
       "7     split xfonts* out of xfree86* . one upload fo...   \n",
       "8                         yes . same binary package .    \n",
       "9    i 'm not suggest all - only the ones you modif...   \n",
       "10   afternoon all not entirely relate to warty , b...   \n",
       "11    you might want to know that thinice in warty ...   \n",
       "12         can i file the panel not link to eds ? : )    \n",
       "13    i just restart x and now nautilus wo n't show...   \n",
       "14    be it a know bug that g-s-t do n't know what ...   \n",
       "15    arse . xt-dev ? i add libx11-dev so just libx...   \n",
       "16                           be away , you say ? nope    \n",
       "17    you 'll be glad to know i 've fix my miss arr...   \n",
       "18    should g2 in ubuntu do the magic dont-focus-w...   \n",
       "19   interest grub-install work with / be ext3 , fa...   \n",
       "20                               2.8 be fix them iirc    \n",
       "21    tell me to come here suggest thursday as a go...   \n",
       "22                                      thats the one    \n",
       "23    i 've get a better one now too , give me a mi...   \n",
       "24    i think experimental be get 2.8 too let him w...   \n",
       "25    i want it on in sarge too but nobody else agree    \n",
       "26               and because python give mark a woody    \n",
       "27    and i think we be a `` pant off '' kind of co...   \n",
       "28    there be an accompany irc conversation to tha...   \n",
       "29    notwarty , hth , hand , kthxbye < g > everyon...   \n",
       "..                                                 ...   \n",
       "170                          this mornings run fine .    \n",
       "171   do - i assume that you 'll poke elmo to fresh...   \n",
       "172  know bug . check bugs.debian.org/src : xfree86...   \n",
       "173  setuid /bin/mount be go to be disable in favou...   \n",
       "174                                    i would say so    \n",
       "175   i will accept responsibility for the addition...   \n",
       "176  we need to make sure that cdroms ( and other r...   \n",
       "177                                               yes    \n",
       "178   sysvinit usplash will also check for it and n...   \n",
       "179     if it work , can you patch it and upload it ?    \n",
       "180                                                np    \n",
       "181   pmount just wrap mount , so long as you be a....   \n",
       "182   i 'll double check that thats the whole reaso...   \n",
       "183   thats a little difficult with the way it be c...   \n",
       "184             what do the fb corruption look like ?    \n",
       "185                                   which package ?    \n",
       "186   they build 'em sturdy down there , apparently...   \n",
       "187                               try apt-get clean ?    \n",
       "188   that issue should n't affect upgrade from woo...   \n",
       "189   one difference be that some pcs have a line o...   \n",
       "190   i think we nail the issue ( executable permis...   \n",
       "191   hmm , good point nono , ( b ) give the cd to ...   \n",
       "192                                              poke    \n",
       "193   yeah , series 1 h8 that debconf should be con...   \n",
       "194  you ping the other day ? ( i only just get to ...   \n",
       "195     right click on the desktop be not work for me    \n",
       "196  anybody else see fb corruption during boot ? g...   \n",
       "197                        no , it 's my home desktop    \n",
       "198  that typically mean that apt be see multiple p...   \n",
       "199   stock warty apt-get install `debootstrap -- p...   \n",
       "\n",
       "                                               UserTwo  \n",
       "0     it would be very easy to keep a hash db of me...  \n",
       "1                                           version ?   \n",
       "2             not yet it be cover by your insurance ?   \n",
       "3     you will be move into your house soon ? post ...  \n",
       "4     not particularly urgent , but a policy violat...  \n",
       "5                                                  ok   \n",
       "6                                           context ?   \n",
       "7                 split the source package you mean ?   \n",
       "8     i would prefer to avoid it at this stage . th...  \n",
       "9     ok , it sound like you 're agree with me , th...  \n",
       "10                                               here   \n",
       "11    and apparently gnome be suddently almost perf...  \n",
       "12    be you use alt ? or the windows key ? wait fo...  \n",
       "13    do you think we have any interest to have hal...  \n",
       "14    somebody should really kick that guy *hard* i...  \n",
       "15    we have plan to speak about menu organisation...  \n",
       "16    the warty repository ok , fine . thanks nice ...  \n",
       "17    i 've upload the gnome-vfs without hal suppor...  \n",
       "18   we 'll have a bof about this so you 're come t...  \n",
       "19    more like osx than debian ; ) we have a selec...  \n",
       "20                pong vino will be in enjoy ubuntu ?   \n",
       "21    we freeze versions a while back : ) you come ...  \n",
       "22    so i saw your email with the mockup at the ai...  \n",
       "23    aha ! no , the gui thingy it 's more wizardy ...  \n",
       "24      we call it 'universe ' ; ) haha ooh , totally   \n",
       "25   i fully endorse this suggestion < /quimby > ho...  \n",
       "26    i 'm not sure if we 're mean to talk about th...  \n",
       "27    mono 1.0 ? dude , that 's go to be a barrel o...  \n",
       "28    but debian/ be also part of diff.gz ... you c...  \n",
       "29    which ? the best feature of the new imac be t...  \n",
       "..                                                 ...  \n",
       "170   yep , it be . could you add hurd-i386 amd64 t...  \n",
       "171  do n't know i have to i 'll remind him next ti...  \n",
       "172   i 'm wait approval to join the sounder list ....  \n",
       "173             under all circumstances , i take it ?   \n",
       "174                                                do   \n",
       "175   indeed . it 's a recommends : thing , fix act...  \n",
       "176   presumably i should make the same quietinit- ...  \n",
       "177   do ( be the actual option change in sysvinit ...  \n",
       "178   the patch as post be wrong , but i 've test w...  \n",
       "179       sure , let 's make it tomorrow though : - )   \n",
       "180   hm , that eject change be go to need some tes...  \n",
       "181                       that 's not my point at all   \n",
       "182   maybe just fall back to something like the ol...  \n",
       "183   do n't have to be exact by any mean be it imp...  \n",
       "184   lot of dot and squggles , text be partly read...  \n",
       "185                                               ^^^   \n",
       "186   over here they install fine , but then apt wa...  \n",
       "187                               that fix it , thank   \n",
       "188   pitti : i 'm in group plugdev , it 's still n...  \n",
       "189   removable devices have be bad for me , but it...  \n",
       "190   it 's copy into their cache already , be n't ...  \n",
       "191  if we be use grub only , should we turn off do...  \n",
       "192   try to remember if i ask if you be ppc or i38...  \n",
       "193                     what kind of puppies ? awesome  \n",
       "194   i ping a few minutes ago do you make a usplas...  \n",
       "195   nautilus manage the desktop ? the icons be di...  \n",
       "196                                which video card ?   \n",
       "197  i remember fb break on that laptop and iirc it...  \n",
       "198   there 's something definitely screwy with the...  \n",
       "199   what kernel be you run ? # 268154 , yeah , re...  \n",
       "\n",
       "[200 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df_merged = pd.DataFrame(df_train['Context'].map(str) + df_train['Utterance'])\n",
    "userOne, userTwo = get_user_arrays(df_train)\n",
    "df_turns = pd.DataFrame({'UserOne': userOne, 'UserTwo': userTwo})\n",
    "df_turns.head(200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sentence-Based DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'i think we could import the old comment via rsync , but from there we need to go via email . I think it be easier than cache the status on each bug and than import bits here and there '"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userOne[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-a5f57e047c74>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mdecoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mencoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mencoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_sentences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muserOne\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muserTwo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'done'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-a5f57e047c74>\u001b[0m in \u001b[0;36mget_sentences\u001b[0;34m(userOne, userTwo)\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32massert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muserOne\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muserTwo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muserOne\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0mone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msent_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muserOne\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m         \u001b[0mone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0ms\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0ms\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'.'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mtwo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msent_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muserTwo\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/nltk/tokenize/__init__.py\u001b[0m in \u001b[0;36msent_tokenize\u001b[0;34m(text, language)\u001b[0m\n\u001b[1;32m     92\u001b[0m     \"\"\"\n\u001b[1;32m     93\u001b[0m     \u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'tokenizers/punkt/{0}.pickle'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlanguage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;31m# Standard word tokenizer.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/nltk/tokenize/punkt.py\u001b[0m in \u001b[0;36mtokenize\u001b[0;34m(self, text, realign_boundaries)\u001b[0m\n\u001b[1;32m   1235\u001b[0m         \u001b[0mGiven\u001b[0m \u001b[0ma\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturns\u001b[0m \u001b[0ma\u001b[0m \u001b[0mlist\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0msentences\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mthat\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1236\u001b[0m         \"\"\"\n\u001b[0;32m-> 1237\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msentences_from_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrealign_boundaries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1238\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1239\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdebug_decisions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/nltk/tokenize/punkt.py\u001b[0m in \u001b[0;36msentences_from_text\u001b[0;34m(self, text, realign_boundaries)\u001b[0m\n\u001b[1;32m   1283\u001b[0m         \u001b[0mfollows\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mperiod\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1284\u001b[0m         \"\"\"\n\u001b[0;32m-> 1285\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspan_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrealign_boundaries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1287\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_slices_from_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/nltk/tokenize/punkt.py\u001b[0m in \u001b[0;36mspan_tokenize\u001b[0;34m(self, text, realign_boundaries)\u001b[0m\n\u001b[1;32m   1274\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrealign_boundaries\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1275\u001b[0m             \u001b[0mslices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_realign_boundaries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1276\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msl\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mslices\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1277\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1278\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msentences_from_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrealign_boundaries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/nltk/tokenize/punkt.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1274\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrealign_boundaries\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1275\u001b[0m             \u001b[0mslices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_realign_boundaries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1276\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msl\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mslices\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1277\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1278\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msentences_from_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrealign_boundaries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/nltk/tokenize/punkt.py\u001b[0m in \u001b[0;36m_realign_boundaries\u001b[0;34m(self, text, slices)\u001b[0m\n\u001b[1;32m   1314\u001b[0m         \"\"\"\n\u001b[1;32m   1315\u001b[0m         \u001b[0mrealign\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1316\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0msl1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msl2\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_pair_iter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mslices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1317\u001b[0m             \u001b[0msl1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mslice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msl1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrealign\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msl1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msl2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/nltk/tokenize/punkt.py\u001b[0m in \u001b[0;36m_pair_iter\u001b[0;34m(it)\u001b[0m\n\u001b[1;32m    309\u001b[0m     \u001b[0mit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m     \u001b[0mprev\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mel\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mit\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    312\u001b[0m         \u001b[0;32myield\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mprev\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m         \u001b[0mprev\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/nltk/tokenize/punkt.py\u001b[0m in \u001b[0;36m_slices_from_text\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m   1290\u001b[0m             \u001b[0mcontext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mmatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'after_tok'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1291\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext_contains_sentbreak\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1292\u001b[0;31m                 \u001b[0;32myield\u001b[0m \u001b[0mslice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlast_break\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1293\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mmatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'next_tok'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1294\u001b[0m                     \u001b[0;31m# next sentence starts after whitespace\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "def get_sentences(userOne, userTwo):\n",
    "    encoder = []\n",
    "    decoder = []\n",
    "    assert(len(userOne) == len(userTwo))\n",
    "    for i in range(len(userOne)):\n",
    "        one = nltk.sent_tokenize(userOne[i])\n",
    "        one = [s for s in one if s != '.']\n",
    "        two = nltk.sent_tokenize(userTwo[i])\n",
    "        two = [s for s in two if s != '.']\n",
    "        combine = one + two\n",
    "        assert(len(combine) == len(one) + len(two))\n",
    "        if len(combine) % 2 != 0:\n",
    "            combine = combine[:-1]\n",
    "        enc = combine[0::2]\n",
    "        dec = combine[1::2]\n",
    "        assert(len(enc) == len(dec))\n",
    "        encoder.append(enc)\n",
    "        decoder.append(dec)\n",
    "    return encoder, decoder\n",
    "encoder, decoder = get_sentences(userOne, userTwo)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "encoder = [nltk.word_tokenize(s[0]) for s in encoder]\n",
    "decoder = [nltk.word_tokenize(s[0]) for s in decoder]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "max_enc_len = max([len(s) for s in encoder])\n",
    "max_dec_len = max([len(s) for s in decoder])\n",
    "print(max_enc_len)\n",
    "print(max_dec_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Analyzing Sentence Lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "encoder_lengths = [len(s) for s in encoder]\n",
    "decoder_lengths = [len(s) for s in decoder]\n",
    "df_lengths = pd.DataFrame({'EncoderSentLength': encoder_lengths, 'DecoderSentLengths': decoder_lengths})\n",
    "df_lengths.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')\n",
    "plt.rcParams['figure.figsize'] = 9, 5\n",
    "fig, axes = plt.subplots(nrows=1, ncols=2)\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.hist(encoder_lengths)\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.hist(decoder_lengths, color='b')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "save_to_file(\"train_from.txt\", userOne)\n",
    "save_to_file(\"train_to.txt\", userTwo)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_valid has 19560 rows.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Context</th>\n",
       "      <th>Utterance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Any ideas on how lts will be release ? __eou__...</td>\n",
       "      <td>We be talk 12.04 not 10.04 __eou__</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>how much hdd use ubuntu default install ? __eo...</td>\n",
       "      <td>thats why i ask how much be default install ? ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>in my country its nearly the 27th __eou__ when...</td>\n",
       "      <td>thanx __eou__</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>it 's not out __eou__ __eot__ they probabaly b...</td>\n",
       "      <td>wait for many things to be setup __eou__ final...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>be the ext4 drivers stable ? __eou__ __eot__ I...</td>\n",
       "      <td>you sound like it 's update to skynet . ; ) __...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Context  \\\n",
       "0  Any ideas on how lts will be release ? __eou__...   \n",
       "1  how much hdd use ubuntu default install ? __eo...   \n",
       "2  in my country its nearly the 27th __eou__ when...   \n",
       "3  it 's not out __eou__ __eot__ they probabaly b...   \n",
       "4  be the ext4 drivers stable ? __eou__ __eot__ I...   \n",
       "\n",
       "                                           Utterance  \n",
       "0                 We be talk 12.04 not 10.04 __eou__  \n",
       "1  thats why i ask how much be default install ? ...  \n",
       "2                                      thanx __eou__  \n",
       "3  wait for many things to be setup __eou__ final...  \n",
       "4  you sound like it 's update to skynet . ; ) __...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"df_valid has\", len(df_valid), \"rows.\")\n",
    "df_valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "userOne, userTwo = get_user_arrays(df_valid)\n",
    "save_to_file(\"valid_from.txt\", userOne)\n",
    "save_to_file(\"valid_to.txt\", userTwo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "print('done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "855 at 1330067\n",
      "Sentence:\n",
      "  thanks : ) i know , it use .deb , but read my word , they say that nothing like .deb or source be there . that be know to me , so iam think of use alien to build the deb i have see the entire internet , and even the sdl developers tell me , that they do n't exist . lol , what be you say ? there be no .deb yet for wesnoth 1.8 , also , i know i need to compile it , but the source be not available for sdl11-config , i instal them all , no result yes , the older ones be available . sorry , but make the previous version be different . the previous versions of wesnoth do not need all sdl libraries , lol , thank , but that be for wesnoth 1.6.1 , at least 2 years older , but i say 1.8 , anyway . thanks for the help . i be search for a source , and let me compile it . lol , still a formidable game can be make , against age of empires series , : d penumbra if commercial , but o a.d be not , : d a demo be not open-source , and also , the what i like be a rts , like the fan of age of mythology and aok , also , from the screenthots of o a.d , it appear as if it be a jewel to ubuntu . the debate continue , if you consider java too , and i do have them . but , the thing be that , commercial < free < open-source , so flash be better than commercial softwares like penumbra what we want to advocate ubuntu for , be freedom and open-source . so , commercial game be not welcome here . the best way be to use free and open-source softwares . is n't our discussion go a bite offtopic ? also , non-free softwares be mean for profit , which clash with the nature of ubuntu . a mutual profit be always welcome , when it benefit the ideology of the user and the developer . and i have research nicely enough , but you seem to challenge the nature of linux , and also , drag the discussion offtopic . in addition , things like non-free softwares cause a flow of money from the pocket of the users to the developers . but open-source be what allow users to be developers . there be , base on user experience . and ubuntu be the best . fame come from others individual judgements , and people accept ubuntu , since it be the best . also , ubuntu be the most user-friendly linux for pcs . anything *buntu , whether lubuntu , kubuntu or xubuntu be the best . the talk be relate to open-source , but windows be not . also , many people use windows , but they be switch to ubuntu now . and it be ubuntu who challenge microsoft in a bold way . computing world be not only for game . for game , get a xbox or playstation . computer be mainly for learn and development . it be for better use . and in all field , ubuntu be the best . for everyday use , puppy be a joke . ubuntu and puppy 's goals differ . ubuntu jeos can do the same . lubuntu be also nicer than puppy in many ways . in the modern world , people who be in computing , should upgrade their systems . and for low-end systems , any linux with low de can do . why only puppy ? dsl can do the same , and lubuntu too . the `` best '' be count as a basis on `` how many field a thing be better than others '' , and ubuntu be much better in most case . so , it be the best . check out the ubuntu synaptic , and you can get anything like that , low-end system maintainance and so . so , lxde can be use in ubuntu , and so can be icewm , so ubuntu can perform much better than puppy . also , where be the package manager for puppy ? nowhere , ubuntu have strongest support , because check the support of both . nice to know it , but then , community of ubuntu need centralization , not fragmentation . i be among ubuntu sideline developers too ( the default version ) i know , but be it really nicer than synaptic ? no , discussing about puppy be not welcome here . it be he ubuntu support channel . as per my question 's answer , you seem to start debate on the main support channel . please do n't do so . lol , : d it be not so bad , and if you hate google , try www.bing.com , : ) by microsoft , : ) lol , nice \n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "userOne, userTwo = get_user_arrays(df_train)\n",
    "# Regular expressions used to tokenize.\n",
    "_WORD_SPLIT = re.compile(b\"([.,!?\\\"':;)(])\")\n",
    "_DIGIT_RE   = re.compile(br\"\\d\")\n",
    "lengths = np.array([len(t.strip().split()) for t in userOne])\n",
    "\n",
    "max_ind =  lengths.argmax()\n",
    "print(max(lengths), \"at\", max_ind)\n",
    "print(\"Sentence:\\n\", userOne[max_ind])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  1.85629000e+06,   2.48062000e+05,   4.06420000e+04,\n",
       "          8.01600000e+03,   2.06200000e+03,   6.29000000e+02,\n",
       "          1.99000000e+02,   8.00000000e+01,   3.90000000e+01,\n",
       "          1.50000000e+01]),\n",
       " array([   0. ,   39.3,   78.6,  117.9,  157.2,  196.5,  235.8,  275.1,\n",
       "         314.4,  353.7,  393. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAD8CAYAAACyyUlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF/RJREFUeJzt3X+sX3Wd5/Hna1thjL8ocpcQClvUzm6QzFboIptRw8oO\nFJxY3DBuyWToOMTqCsmY2c1Y1mRxcUxwNg67JIrBpUtxlR8DujRjXewCGbN/8KNIhaIiFyyhTaUd\nijCzzKDAe//4fq58e733tr2f2/tt5flITr7n+z6f8znv72muL77nnHtNVSFJUo9/NOoGJEmHP8NE\nktTNMJEkdTNMJEndDBNJUjfDRJLUzTCRJHUzTCRJ3QwTSVK3haNuYL4cc8wxtWTJklG3IUmHlQce\neOBvqmpsX+NeM2GyZMkSNm/ePOo2JOmwkuTJ/RnnZS5JUjfDRJLUzTCRJHUzTCRJ3QwTSVI3w0SS\n1M0wkSR1M0wkSd0ME0lSt9fMb8D3WrL2WyM57rYrPzCS40rSgfCbiSSpm2EiSepmmEiSuhkmkqRu\nhokkqZthIknqZphIkrrtM0ySrEuyK8nWodrNSba0ZVuSLa2+JMnfD2378tA+pyV5OMl4kquTpNWP\nTrIpyWPtdVGrp40bT/JQklOH5lrdxj+WZPVcnhBJ0oHbn28m1wMrhgtV9W+rallVLQNuA74xtPnx\niW1V9fGh+jXAR4GlbZmYcy1wZ1UtBe5s7wHOHRq7pu1PkqOBy4F3A6cDl08EkCRpNPYZJlX1XWDP\nVNvat4sPAzfONEeS44A3V9U9VVXADcD5bfNKYH1bXz+pfkMN3AMc1eY5B9hUVXuq6llgE5PCTpI0\nv3rvmbwXeLqqHhuqnZTkwSR/neS9rXY8sH1ozPZWAzi2qna29Z8Cxw7t89QU+0xXlySNSO/f5rqQ\nvb+V7AROrKpnkpwG/K8k79zfyaqqklRnT7+UZA2DS2SceOKJczWtJGmSWX8zSbIQ+DfAzRO1qnqx\nqp5p6w8AjwO/CewAFg/tvrjVAJ5ul68mLoftavUdwAlT7DNd/VdU1bVVtbyqlo+Njc3mY0qS9kPP\nZa5/Dfyoqn55+SrJWJIFbf1tDG6eP9EuYz2f5Ix2n+Ui4Pa22wZg4oms1ZPqF7Wnus4Anmvz3AGc\nnWRRu/F+dqtJkkZkn5e5ktwInAkck2Q7cHlVXQes4ldvvL8PuCLJL4BXgI9X1cTN+08weDLs9cC3\n2wJwJXBLkouBJxnc0AfYCJwHjAMvAB8BqKo9ST4L3N/GXTF0DEnSCOwzTKrqwmnqfzhF7TYGjwpP\nNX4zcMoU9WeAs6aoF3DJNHOtA9bN1Lckaf74G/CSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZth\nIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZth\nIknqts8wSbIuya4kW4dqn0myI8mWtpw3tO2yJONJHk1yzlB9RauNJ1k7VD8pyb2tfnOSI1r9yPZ+\nvG1fsq9jSJJGY3++mVwPrJiiflVVLWvLRoAkJwOrgHe2fb6UZEGSBcAXgXOBk4EL21iAz7e53gE8\nC1zc6hcDz7b6VW3ctMc4sI8tSZpL+wyTqvousGc/51sJ3FRVL1bVT4Bx4PS2jFfVE1X1c+AmYGWS\nAO8Hbm37rwfOH5prfVu/FTirjZ/uGJKkEem5Z3JpkofaZbBFrXY88NTQmO2tNl39rcDPquqlSfW9\n5mrbn2vjp5vrVyRZk2Rzks27d++e3aeUJO3TbMPkGuDtwDJgJ/CFOetoDlXVtVW1vKqWj42Njbod\nSfq1Naswqaqnq+rlqnoF+AqvXmbaAZwwNHRxq01XfwY4KsnCSfW95mrb39LGTzeXJGlEZhUmSY4b\nevshYOJJrw3AqvYk1knAUuA+4H5gaXty6wgGN9A3VFUBdwMXtP1XA7cPzbW6rV8A3NXGT3cMSdKI\nLNzXgCQ3AmcCxyTZDlwOnJlkGVDANuBjAFX1SJJbgB8ALwGXVNXLbZ5LgTuABcC6qnqkHeJTwE1J\n/gx4ELiu1a8DvppknMEDAKv2dQxJ0mhk8B/7v/6WL19emzdvnvX+S9Z+aw672X/brvzASI4rSQBJ\nHqiq5fsa52/AS5K6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJ\nJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqds+wyTJuiS7kmwdqv2XJD9K\n8lCSbyY5qtWXJPn7JFva8uWhfU5L8nCS8SRXJ0mrH51kU5LH2uuiVk8bN96Oc+rQXKvb+MeSrJ7L\nEyJJOnD7883kemDFpNom4JSq+i3gx8BlQ9ser6plbfn4UP0a4KPA0rZMzLkWuLOqlgJ3tvcA5w6N\nXdP2J8nRwOXAu4HTgcsnAkiSNBr7DJOq+i6wZ1LtO1X1Unt7D7B4pjmSHAe8uaruqaoCbgDOb5tX\nAuvb+vpJ9Rtq4B7gqDbPOcCmqtpTVc8yCLbJYSdJmkdzcc/kj4BvD70/KcmDSf46yXtb7Xhg+9CY\n7a0GcGxV7WzrPwWOHdrnqSn2ma7+K5KsSbI5yebdu3cf4MeSJO2vrjBJ8mngJeBrrbQTOLGq3gX8\nCfD1JG/e3/nat5bq6WnSfNdW1fKqWj42NjZX00qSJpl1mCT5Q+B3gd9vIUBVvVhVz7T1B4DHgd8E\ndrD3pbDFrQbwdLt8NXE5bFer7wBOmGKf6eqSpBGZVZgkWQH8KfDBqnphqD6WZEFbfxuDm+dPtMtY\nzyc5oz3FdRFwe9ttAzDxRNbqSfWL2lNdZwDPtXnuAM5OsqjdeD+71SRJI7JwXwOS3AicCRyTZDuD\nJ6kuA44ENrUnfO9pT269D7giyS+AV4CPV9XEzftPMHgy7PUM7rFM3Ge5ErglycXAk8CHW30jcB4w\nDrwAfASgqvYk+Sxwfxt3xdAxJEkjsM8wqaoLpyhfN83Y24Dbptm2GThlivozwFlT1Au4ZJq51gHr\npu9akjSf/A14SVI3w0SS1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdTNMJEndDBNJUjfDRJLUzTCR\nJHUzTCRJ3QwTSVI3w0SS1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdduvMEmyLsmuJFuHakcn2ZTk\nsfa6qNWT5Ook40keSnLq0D6r2/jHkqweqp+W5OG2z9VJMttjSJLm3/5+M7keWDGptha4s6qWAne2\n9wDnAkvbsga4BgbBAFwOvBs4Hbh8IhzamI8O7bdiNseQJI3GfoVJVX0X2DOpvBJY39bXA+cP1W+o\ngXuAo5IcB5wDbKqqPVX1LLAJWNG2vbmq7qmqAm6YNNeBHEOSNAI990yOraqdbf2nwLFt/XjgqaFx\n21ttpvr2KeqzOcZekqxJsjnJ5t27dx/AR5MkHYg5uQHfvlHUXMw1l8eoqmuranlVLR8bGztInUmS\nesLk6YlLS+11V6vvAE4YGre41WaqL56iPptjSJJGoCdMNgATT2StBm4fql/Unrg6A3iuXaq6Azg7\nyaJ24/1s4I627fkkZ7SnuC6aNNeBHEOSNAIL92dQkhuBM4Fjkmxn8FTWlcAtSS4GngQ+3IZvBM4D\nxoEXgI8AVNWeJJ8F7m/jrqiqiZv6n2DwxNjrgW+3hQM9hiRpNPYrTKrqwmk2nTXF2AIumWaedcC6\nKeqbgVOmqD9zoMeQJM0/fwNektTNMJEkdTNMJEndDBNJUjfDRJLUzTCRJHUzTCRJ3QwTSVI3w0SS\n1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdTNMJEndDBNJUjfDRJLUzTCRJHWbdZgk+adJtgwtzyf5\nZJLPJNkxVD9vaJ/LkowneTTJOUP1Fa02nmTtUP2kJPe2+s1Jjmj1I9v78bZ9yWw/hySp36zDpKoe\nraplVbUMOA14Afhm23zVxLaq2giQ5GRgFfBOYAXwpSQLkiwAvgicC5wMXNjGAny+zfUO4Fng4la/\nGHi21a9q4yRJIzJXl7nOAh6vqidnGLMSuKmqXqyqnwDjwOltGa+qJ6rq58BNwMokAd4P3Nr2Xw+c\nPzTX+rZ+K3BWGy9JGoG5CpNVwI1D7y9N8lCSdUkWtdrxwFNDY7a32nT1twI/q6qXJtX3mqttf66N\nlySNQHeYtPsYHwT+spWuAd4OLAN2Al/oPcZsJVmTZHOSzbt37x5VG5L0a28uvpmcC3yvqp4GqKqn\nq+rlqnoF+AqDy1gAO4AThvZb3GrT1Z8BjkqycFJ9r7na9re08XupqmuranlVLR8bG+v+oJKkqc1F\nmFzI0CWuJMcNbfsQsLWtbwBWtSexTgKWAvcB9wNL25NbRzC4ZLahqgq4G7ig7b8auH1ortVt/QLg\nrjZekjQCC/c9ZHpJ3gD8DvCxofKfJ1kGFLBtYltVPZLkFuAHwEvAJVX1cpvnUuAOYAGwrqoeaXN9\nCrgpyZ8BDwLXtfp1wFeTjAN7GASQJGlEusKkqv4fk258V9UfzDD+c8DnpqhvBDZOUX+CVy+TDdf/\nAfi9WbQsSToI/A14SVI3w0SS1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdTNMJEndDBNJUjfDRJLU\nzTCRJHUzTCRJ3QwTSVI3w0SS1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdesOkyTbkjycZEuSza12\ndJJNSR5rr4taPUmuTjKe5KEkpw7Ns7qNfyzJ6qH6aW3+8bZvZjqGJGn+zdU3k39VVcuqanl7vxa4\ns6qWAne29wDnAkvbsga4BgbBAFwOvBs4Hbh8KByuAT46tN+KfRxDkjTPDtZlrpXA+ra+Hjh/qH5D\nDdwDHJXkOOAcYFNV7amqZ4FNwIq27c1VdU9VFXDDpLmmOoYkaZ7NRZgU8J0kDyRZ02rHVtXOtv5T\n4Ni2fjzw1NC+21ttpvr2KeozHUOSNM8WzsEc76mqHUn+MbApyY+GN1ZVJak5OM60pjtGC7c1ACee\neOLBbEGSXtO6v5lU1Y72ugv4JoN7Hk+3S1S0111t+A7ghKHdF7faTPXFU9SZ4RjDvV1bVcuravnY\n2FjPx5QkzaArTJK8IcmbJtaBs4GtwAZg4oms1cDtbX0DcFF7qusM4Ll2qeoO4Owki9qN97OBO9q2\n55Oc0Z7iumjSXFMdQ5I0z3ovcx0LfLM9rbsQ+HpV/e8k9wO3JLkYeBL4cBu/ETgPGAdeAD4CUFV7\nknwWuL+Nu6Kq9rT1TwDXA68Hvt0WgCunOYYkaZ51hUlVPQH88ynqzwBnTVEv4JJp5loHrJuivhk4\nZX+PIUmaf/4GvCSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maY\nSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkrrNOkySnJDk7iQ/SPJIkj9u\n9c8k2ZFkS1vOG9rnsiTjSR5Ncs5QfUWrjSdZO1Q/Kcm9rX5zkiNa/cj2frxtXzLbzyFJ6tfzzeQl\n4N9X1cnAGcAlSU5u266qqmVt2QjQtq0C3gmsAL6UZEGSBcAXgXOBk4ELh+b5fJvrHcCzwMWtfjHw\nbKtf1cZJkkZk1mFSVTur6ntt/W+BHwLHz7DLSuCmqnqxqn4CjAOnt2W8qp6oqp8DNwErkwR4P3Br\n2389cP7QXOvb+q3AWW28JGkE5uSeSbvM9C7g3la6NMlDSdYlWdRqxwNPDe22vdWmq78V+FlVvTSp\nvtdcbftzbbwkaQQW9k6Q5I3AbcAnq+r5JNcAnwWqvX4B+KPe48yytzXAGoATTzxxFC10W7L2WyM5\n7rYrPzCS40o6PHV9M0nyOgZB8rWq+gZAVT1dVS9X1SvAVxhcxgLYAZwwtPviVpuu/gxwVJKFk+p7\nzdW2v6WN30tVXVtVy6tq+djYWM9HlSTNoOdprgDXAT+sqr8Yqh83NOxDwNa2vgFY1Z7EOglYCtwH\n3A8sbU9uHcHgJv2GqirgbuCCtv9q4PahuVa39QuAu9p4SdII9Fzm+m3gD4CHk2xptf/I4GmsZQwu\nc20DPgZQVY8kuQX4AYMnwS6pqpcBklwK3AEsANZV1SNtvk8BNyX5M+BBBuFFe/1qknFgD4MAkiSN\nyKzDpKr+LzDVE1QbZ9jnc8DnpqhvnGq/qnqCVy+TDdf/Afi9A+lXknTw+BvwkqRuhokkqZthIknq\nZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknq\nZphIkroZJpKkboaJJKlbz/8HvH6NLVn7rZEde9uVHxjZsSXNzmH9zSTJiiSPJhlPsnbU/UjSa9Vh\nGyZJFgBfBM4FTgYuTHLyaLuSpNemwzZMgNOB8ap6oqp+DtwErBxxT5L0mnQ43zM5Hnhq6P124N0j\n6kVzaFT3a7xXI83e4Rwm+5RkDbCmvf27JI92THcM8Df9Xc25Q7UvOMx6y+dH1MneDqtzdgixtwO3\nv339k/2Z7HAOkx3ACUPvF7faL1XVtcC1c3GwJJuravlczDWXDtW+wN5m41DtC+xttg7V3ua6r8P5\nnsn9wNIkJyU5AlgFbBhxT5L0mnTYfjOpqpeSXArcASwA1lXVIyNuS5Jekw7bMAGoqo3Axnk63Jxc\nLjsIDtW+wN5m41DtC+xttg7V3ua0r1TVXM4nSXoNOpzvmUiSDhGGyT4can+yJcm2JA8n2ZJkc6sd\nnWRTksfa66J56mVdkl1Jtg7VpuwlA1e38/hQklPnua/PJNnRztuWJOcNbbus9fVoknMOVl/tWCck\nuTvJD5I8kuSPW32k522GvkZ+3pL8RpL7kny/9fafW/2kJPe2Hm5uD+KQ5Mj2frxtXzKC3q5P8pOh\n87as1eft56Adb0GSB5P8VXt/8M5ZVblMszC4sf848DbgCOD7wMkj7mkbcMyk2p8Da9v6WuDz89TL\n+4BTga376gU4D/g2EOAM4N557uszwH+YYuzJ7d/1SOCk9u+94CD2dhxwalt/E/Dj1sNIz9sMfY38\nvLXP/sa2/jrg3nYubgFWtfqXgX/X1j8BfLmtrwJuPoj/ntP1dj1wwRTj5+3noB3vT4CvA3/V3h+0\nc+Y3k5kdLn+yZSWwvq2vB86fj4NW1XeBPfvZy0rghhq4BzgqyXHz2Nd0VgI3VdWLVfUTYJzBv/tB\nUVU7q+p7bf1vgR8y+GsOIz1vM/Q1nXk7b+2z/117+7q2FPB+4NZWn3zOJs7lrcBZSTLPvU1n3n4O\nkiwGPgD89/Y+HMRzZpjMbKo/2TLTD9h8KOA7SR7I4Df8AY6tqp1t/afAsaNpbcZeDoVzeWm7tLBu\n6FLgyPpqlxLexeC/Zg+Z8zapLzgEzlu7XLMF2AVsYvBN6GdV9dIUx/9lb237c8Bb56u3qpo4b59r\n5+2qJEdO7m2KvufafwX+FHilvX8rB/GcGSaHn/dU1akM/lryJUneN7yxBt9TD4lH9A6lXoBrgLcD\ny4CdwBdG2UySNwK3AZ+squeHt43yvE3R1yFx3qrq5apaxuAvXZwO/LNR9DGVyb0lOQW4jEGP/wI4\nGvjUfPaU5HeBXVX1wHwd0zCZ2T7/ZMt8q6od7XUX8E0GP1hPT3xVbq+7RtfhtL2M9FxW1dPth/4V\n4Cu8eklm3vtK8joG/4P9tar6RiuP/LxN1dehdN5aPz8D7gb+JYNLRBO/Kzd8/F/21ra/BXhmHntb\n0S4bVlW9CPwP5v+8/TbwwSTbGFyefz/w3ziI58wwmdkh9SdbkrwhyZsm1oGzga2tp9Vt2Grg9tF0\nCDP0sgG4qD3Ncgbw3NBlnYNu0nXpDzE4bxN9rWpPs5wELAXuO4h9BLgO+GFV/cXQppGet+n6OhTO\nW5KxJEe19dcDv8Pgns7dwAVt2ORzNnEuLwDuat/25qu3Hw39h0EY3JcYPm8H/d+zqi6rqsVVtYTB\n/27dVVW/z8E8Z3P99MCv28Lg6YsfM7hG++kR9/I2Bk/QfB94ZKIfBtc27wQeA/4PcPQ89XMjg0sf\nv2Bw/fXi6Xph8PTKF9t5fBhYPs99fbUd96H2g3Pc0PhPt74eBc49yOfsPQwuYT0EbGnLeaM+bzP0\nNfLzBvwW8GDrYSvwn4Z+Hu5jcPP/L4EjW/032vvxtv1tI+jtrnbetgL/k1ef+Jq3n4OhHs/k1ae5\nDto58zfgJUndvMwlSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKnb/weEUcDaa1M/\nHAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f3f44d83e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.hist(sorted(lengths)[:-20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2134602 out of 2156054 (0.9900503419673162\\%)\n"
     ]
    }
   ],
   "source": [
    "n_under_20 = sum([1 if l < 100 else 0 for l in lengths])\n",
    "print(n_under_20, \"out of\", len(lengths), \"({}\\%)\".format(float(n_under_20)/len(lengths)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_lengths = pd.DataFrame(lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2.156054e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.171190e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.074356e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.600000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.800000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.550000e+02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0\n",
       "count  2.156054e+06\n",
       "mean   2.171190e+01\n",
       "std    2.074356e+01\n",
       "min    0.000000e+00\n",
       "25%    8.000000e+00\n",
       "50%    1.600000e+01\n",
       "75%    2.800000e+01\n",
       "max    8.550000e+02"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_lengths.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Relationship between Accuracy, Loss, and Others\n",
    "\n",
    "$$\n",
    "\\text{Loss} = - \\frac{1}{N} \\sum_{i = 1}^{N} \\ln\\left(p_{target_i}\\right)\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss for uniformly random guessing is 10.5966347331\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel/__main__.py:16: RuntimeWarning: divide by zero encountered in true_divide\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fdd93e5bba8>]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGcpJREFUeJzt3WtwW/d55/HvgwtBUhQpyaRkWaJ1sdVUqt34wrqOneyk\ntZOVva2VaZvE2qZpU0/0Jm7dNtOOu+1ku94XbS6TpGlcN5o06yaTtdfxZrKaVK2cOG6SSX0RHbuy\nJVkyY0uWZF0o6kaRIkEAz744ByRIASQkgYTOwe8zg8G5/IHzh4/8w58PzsXcHRERiZdEvTsgIiK1\np3AXEYkhhbuISAwp3EVEYkjhLiISQwp3EZEYUriLiMSQwl1EJIYU7iIiMZSq14Y7Ozt95cqV9dq8\niEgkvfjii8fdvWumdnUL95UrV9Lb21uvzYuIRJKZ7a+mncoyIiIxpHAXEYkhhbuISAwp3EVEYkjh\nLiISQzOGu5l9zcyOmdmrFdabmX3JzPrMbIeZ3VT7boqIyIWoZuT+KLB+mvV3AWvCxybgkUvvloiI\nXIoZw93dfwScmKbJBuDrHngOWGBmS2vVwam27zvB57btIZcvzNYmREQirxY192XAgZL5g+Gy85jZ\nJjPrNbPe/v7+i9rYS2+d5MvP9DGSU7iLiFQypz+ouvtmd+9x956urhnPni0rk0oCMDqWr2XXRERi\npRbhfgjoLplfHi6bFZlU0OVRjdxFRCqqRbhvAT4aHjVzK3Da3Q/X4H3LyqQV7iIiM5nxwmFm9hjw\nXqDTzA4C/x1IA7j7PwBbgbuBPmAY+NhsdRZKyjI5lWVERCqZMdzdfeMM6x34RM16NIPxssyYRu4i\nIpVE7gzViZG7wl1EpJLohXtYc88q3EVEKopeuI8fLaOau4hIJREMd5VlRERmEsFw18hdRGQm0Qv3\ntI6WERGZSfTCXWUZEZEZRTDcVZYREZlJdMNdZRkRkYoiF+6pZIJkwlSWERGZRuTCHYLRu8oyIiKV\nRTjcNXIXEakkouGeVM1dRGQa0Qz3tMoyIiLTiWa4qywjIjKtSIZ7k8JdRGRakQz3TCqpsoyIyDQi\nGu4J/aAqIjKN6Ia7yjIiIhVFNNxVlhERmU40wz2tkbuIyHSiGe6quYuITCui4a6yjIjIdCIa7irL\niIhMJ5rhrpq7iMi0ohnuqST5gpPLK+BFRMqJaLgXb7WncBcRKUfhLiISQ9EM93QS0E2yRUQqiWa4\nhyP3rEbuIiJlRTTciyN3hbuISDlVhbuZrTezPWbWZ2YPlll/tZk9Y2YvmdkOM7u79l2dMF5z11mq\nIiJlzRjuZpYEHgbuAtYBG81s3ZRmfwk84e43AvcCf1/rjpbKpIs/qKrmLiJSTjUj91uAPnd/w92z\nwOPAhiltHGgPpzuAt2vXxfOpLCMiMr1qwn0ZcKBk/mC4rNRfAR8xs4PAVuAPyr2RmW0ys14z6+3v\n77+I7gYmDoXUyF1EpJxa/aC6EXjU3ZcDdwPfMLPz3tvdN7t7j7v3dHV1XfTGxssyqrmLiJRVTbgf\nArpL5peHy0rdBzwB4O7PAs1AZy06WI7KMiIi06sm3LcDa8xslZk1EfxgumVKm7eAOwDMbC1BuF98\n3WUGKsuIiExvxnB39xxwP7AN2E1wVMxOM3vIzO4Jm30S+LiZ/QfwGPB77u6z1WldfkBEZHqpahq5\n+1aCH0pLl32qZHoXcHttu1bZ+OUHVHMXESkromeoqiwjIjKdSIZ7KmGYqSwjIlJJJMPdzHSrPRGR\naUQy3CG8SfaYyjIiIuVEONw1chcRqSS64a6bZIuIVBTdcE8ldbSMiEgFEQ73hI5zFxGpINrhrrKM\niEhZEQ53lWVERCqJbrjrB1URkYqiG+6quYuIVBThcFdZRkSkkgiHu8oyIiKVRDfcVXMXEakouuGu\na8uIiFQU4XDXyF1EpJIIh3uSXMHJF2btbn4iIpEV3XBPB13PavQuInKe6Ia7brUnIlJRhMM9vEm2\nRu4iIueJcLiHI3edpSoicp7ohntaZRkRkUqiG+4qy4iIVBThcNfIXUSkkuiHu2ruIiLniWy4N42P\n3BXuIiJTRTbcJ2ruKsuIiEwV3XBPa+QuIlJJdMNdNXcRkYqqCnczW29me8ysz8werNDmQ2a2y8x2\nmtn/rm03z6eyjIhIZamZGphZEngYeB9wENhuZlvcfVdJmzXAnwO3u/tJM1s8Wx0uUllGRKSyakbu\ntwB97v6Gu2eBx4ENU9p8HHjY3U8CuPux2nbzfBkdLSMiUlE14b4MOFAyfzBcVurngJ8zs5+Y2XNm\ntr5WHaykKVmsuassIyIy1YxlmQt4nzXAe4HlwI/M7Hp3P1XayMw2AZsArr766kvaoJnpbkwiIhVU\nM3I/BHSXzC8Pl5U6CGxx9zF3fxPYSxD2k7j7Znfvcfeerq6ui+3zOIW7iEh51YT7dmCNma0ysybg\nXmDLlDbfIRi1Y2adBGWaN2rYz7Iy6aSOlhERKWPGcHf3HHA/sA3YDTzh7jvN7CEzuydstg0YMLNd\nwDPAn7r7wGx1uiiTSug4dxGRMqqqubv7VmDrlGWfKpl24E/Cx5xRWUZEpLzInqEKwYlMKsuIiJwv\n2uGe1shdRKScaIe7au4iImVFPNxVlhERKSfi4a6yjIhIOdEO93RS4S4iUka0wz2VIKtwFxE5T+TD\nXTV3EZHzRTzckzpaRkSkjGiHu45zFxEpK9rhnkqQzRcoFLzeXRERuaxEPNyD+6hm8xq9i4iUinS4\nNxVvtae6u4jIJJEO94n7qOqIGRGRUjEJd43cRURKRTrc5zcHl6M/fW6szj0REbm8RDrcu+ZnADh+\ndrTOPRERubxEO9zbmgHoH1S4i4iUinS4d85vAqBfI3cRkUkiHe6tTSnaMimN3EVEpoh0uENQd1e4\ni4hMFv1wb1O4i4hMFflw75zfpJq7iMgUkQ93jdxFRM4X/XCfn2FwJMfImC5BICJSFItwB53IJCJS\nKjbhrtKMiMiE6Ie7zlIVETlP9MO9OHJXWUZEZFzkw/2KtvASBBq5i4iMi3y4p5MJFs1rUriLiJSo\nKtzNbL2Z7TGzPjN7cJp2v2lmbmY9tevizHSsu4jIZDOGu5klgYeBu4B1wEYzW1em3XzgAeD5Wndy\nJjpLVURksmpG7rcAfe7+hrtngceBDWXa/U/g08BIDftXFY3cRUQmqybclwEHSuYPhsvGmdlNQLe7\n/3MN+1a14pUh3b0emxcRuexc8g+qZpYAPg98soq2m8ys18x6+/v7L3XT47rmZxjNFTg7mqvZe4qI\nRFk14X4I6C6ZXx4uK5oPXAf8m5ntA24FtpT7UdXdN7t7j7v3dHV1XXyvp9BZqiIik1UT7tuBNWa2\nysyagHuBLcWV7n7a3TvdfaW7rwSeA+5x995Z6XEZOktVRGSyGcPd3XPA/cA2YDfwhLvvNLOHzOye\n2e5gNXSWqojIZKlqGrn7VmDrlGWfqtD2vZferQujsoyIyGSRP0MVYEFLmlTCFO4iIqFYhHsiYXTq\nWHcRkXGxCHfQWaoiIqViE+46S1VEZEJ8wn2+wl1EpChW4T4wlKVQ0CUIRETiE+5tGfIF5+Rwtt5d\nERGpu/iE+/zwLFX9qCoiEp9wv7IjOJHp0Mlzde6JiEj9xSbc1yyZD8Duw2fq3BMRkfqLTbi3N6e5\nelEruw8P1rsrIiJ1F5twB1i3tJ1dGrmLiMQr3NcubWffwJBu2iEiDS9W4b7uqnbcYc8Rjd5FpLHF\nLtwBdqnuLiINLlbhflVHMx0taXa9rZG7iDS2WIW7melHVRERYhbuEJRm9hw5Q17XmBGRBha7cF+7\ntJ2RsQJvHh+qd1dEROomduG+bmnxR1WVZkSkccUu3K9d3EY6afpRVUQaWuzCvSmVYM3i+Rq5i0hD\ni124Q/Cjqi4gJiKNLJ7hvrSd/sFRjg2O1LsrIiJ1EctwX1v8UVV1dxFpULEM9+uXd5BOGj/pO17v\nroiI1EUsw70tk+K2azp5atdR3HUyk4g0nliGO8D7f2EJ+weGef3Y2Xp3RURkzsU23O9cuwSAp3Ye\nqXNPRETmXmzDfUl7Mzd0L+CpXUfr3RURkTkX23CHoDSz4+BpDp8+V++uiIjMqarC3czWm9keM+sz\nswfLrP8TM9tlZjvM7GkzW1H7rl64968LSjPf1+hdRBrMjOFuZkngYeAuYB2w0czWTWn2EtDj7r8I\nPAl8ptYdvRjXdLWxunOeSjMi0nCqGbnfAvS5+xvungUeBzaUNnD3Z9x9OJx9Dlhe225eHDPjfeuW\n8OzPBjh9bqze3RERmTPVhPsy4EDJ/MFwWSX3Af9yKZ2qpff/whJyBeeZ147VuysiInOmpj+omtlH\ngB7gsxXWbzKzXjPr7e/vr+WmK7qxeyHdi1p49N/36YQmEWkY1YT7IaC7ZH55uGwSM7sT+AvgHncf\nLfdG7r7Z3Xvcvaerq+ti+nvBEglj03tW8/KBUzz/5ok52aaISL1VE+7bgTVmtsrMmoB7gS2lDczs\nRuArBMF+2dU/PtjTzRXzmnjk335W766IiMyJGcPd3XPA/cA2YDfwhLvvNLOHzOyesNlngTbgW2b2\nspltqfB2ddGcTvKx21fyw739ulKkiDQEq1cduqenx3t7e+dse6eHx7jtb57mjrVL+NLGG+dsuyIi\ntWRmL7p7z0ztYn2GaqmO1jT/9Zev5rs73ubAieGZXyAiEmENE+4A9717NcmE8YXv7613V0REZlVD\nhfuVHc18/D2r+fZPD/HDvXNzKKaISD00VLgD/OEda7imax7/7duvcHY0V+/uiIjMioYL9+Z0ks/8\n1jt5+/Q5PvOvr9W7OyIis6Lhwh3g5hUL+b3bVvL1Z/fzgk5sEpEYashwB/jT//wOrl7Uyh889lPe\nPqXrvYtIvDRsuLc2pdj80ZsZHs3z+49u58yIrhopIvHRsOEO8PNXtvPIR26m79hZPvHNnzKWL9S7\nSyIiNdHQ4Q7w7jWd/PVvXM+PXz/Onz25QwEvIrGQqncHLgcf7Onm2OAon922h1PDWR7+7ZtobdJ/\nGhGJroYfuRd94leu5a9/43p+uLefjZuf4/jZslctFhGJBIV7iY23XM3m3+lhz9FBPvDwT3hx/8l6\nd0lE5KIo3Ke4c90SHt/0LgA+9JVn+dLTr5Mv6A5OIhItCvcybuhewNYH3sOv/+JSPv+9vXzoK8/q\nOvAiEikK9wram9N88d4b+eKHb+DN40P82t/9mL/8ziucHMrWu2siIjPSISEz+MCNy/iVdyzmC9/f\nyzee28+Wl9/m99+9io/dtoqO1nS9uyciUlbD3ImpFvYcGeRzT+3he7uO0pZJ8dF3reB33rWCpR0t\n9e6aiDSIau/EpHC/CLsPn+HLz/Sx9ZXDJMy4c+1iPnLrCm6/ppNEwurdPRGJMYX7HHhrYJhvvrCf\nb/Ue5MRQlivbm7nnhqvYcMNVrFvajpmCXkRqS+E+h0bG8mzbeYQtL7/ND/f2kys4K65o5c61S7hz\n7RJ6Vi4kndRv1yJy6RTudXJyKMu/vHqEp3Yd4d/7BsjmC7RlUty6ehG3X9vJbdd0smZxm8o3InJR\nFO6XgaHRHD9+vZ8fvX6cn/QdZ//AMADtzSl6Vi7i5hULuaF7Adcv76C9WUfeiMjMqg13HQo5i+Zl\nUqy/binrr1sKwIETw7zw5gm27zvBC/tO8IPXjo23Xd05j7VXtbNuafBYs6SNZQtaVLcXkYuicJ9D\n3Yta6V7Uym/evByA08Nj7Dh0iv84cIpXD53hlYOn+ecdh8fbz2tKcu2S+VzTNY9rutpY3TmPFVfM\nY8UVrczLaNeJSGUqy1xmzoyMsefIIHuPDvL60bPsPTrIG/1DHDkzMqldZ1uG7kUtdC9sZfnCFpYt\nbOGqjhauWtDClR3NtDenNOoXiSGVZSKqvTnNL61cxC+tXDRp+dnRHPuOD7F/YJh9A0PsHxji4Mlz\nvHzgFFtfOUxuysXNWpuSXNnezOL2DEvam1k8P8Pi+c10zm+isy1DZ1uGK9qaWNTaREpH8ojEjsI9\nItoyKa5b1sF1yzrOW5cvOMcGR3j71DkOnRrh6OkRDp8e4eiZ4PHSW6c4emaE0Vz5u0wtaE2zaF4Q\n9AvnNbGgJR08t6ZZ0NJER0uaBa1pOlrStDcHz23NKZI64kfksqVwj4Fkwlja0cLSjhZuXlG+jbsz\nOJrj+OAo/YOjDAxlGTg7yvGzWQaGRjk5PMaJs1kOnBjmleExTg5nK34ZFLVlUrQ3p5jfHIT9/OJ0\nJkVbJsm8TIq2TIp54aMtk6S1KVjW0pRkXlPw3NqU1HkAIjWmcG8QZkZ7czDyXt3VVtVrzmXznD43\nxulzY5wazo5Pnz43xuBIjjMjY5w5l+PsaDA/cDbLWwPDDI7mGBwZY2Ss+vvRppNGSzoZhn1qfLol\nnaQ5nG5OJYLndDCdCddlUolgWTpBJjUxn0klyKQTNCWDtsFzOJ9K6DcJiTWFu1TU0hSE6pUdzRf1\n+ly+wFA2z9BojuFsjqHRYHoom2c4m2M4XHcum2d4LB88Z3OcGytwLpvj3FgwPzCUZXQsz7nwMTKW\nv6AvjkqakgmaUuEjnE4njaZUMlxm4bLg0ZQM1qeTCdKpBOlEyXQyaJ9KFtsbqUSCVNLGp4uvTSWD\n16aSwfpUYmJ9MnzPZMLCdeH0lHmRmVQV7ma2HvhbIAl81d3/Zsr6DPB14GZgAPiwu++rbVclalLJ\nBB0tCTpaan+ClrszmiswOlZgNBeEffF5JJcnmwvmR0vmg2XBI1vynM0H68fyPr58LB+sGxkrcHYk\nRzbvZHN5cgVnLFcgmw/aj+UL4WPujjozg1TCwtCfCP/ic2LSfGLS/PjDgi+LhE1+TSJcV9oukTCS\nCYL3smC62C5hpa9h0usTxeeEkTAm3itcbsZ4u+BBSfvi+zM+f970+PaD97bx7QbrrLSNMWm9TXmP\nibYTr436X3YzhruZJYGHgfcBB4HtZrbF3XeVNLsPOOnu15rZvcCngQ/PRodFIPgfrzksy0D9z+51\n9yD4w6DPlYR/rhDMZ/MFcnknVwjW5cP2wbJg+fh0vsBYwSkUJuZzheA1uXyBvDu5vDOWdwoevDZf\nYHxd0C58Ljj5QoG8Q74w0YfRXLC++F7F6eI2C+F8vgCF8D3HH+54uCyud6EsDXyj/BdA8cvGmPqF\nEczblC+a4vMDd6zh19951az2v5qR+y1An7u/EXxgexzYAJSG+wbgr8LpJ4Evm5l5vQ6iF5ljZjZe\ndmk07kHAB0E/+Uti4ouA8S+GQkl7dx//MhpvE35xlL6nT5kulHyxTExPtHOC5YUp06XtoTgNTslr\nwzal2yp48Dkdxr/0PFw2Ph2um/QaJuZLn2fjr9mpqgn3ZcCBkvmDwC9XauPuOTM7DVwBHK9FJ0Xk\n8mVmJMOSilw+5nSYYWabzKzXzHr7+/vnctMiIg2lmnA/BHSXzC8Pl5VtY2YpoIPgh9VJ3H2zu/e4\ne09XV9fF9VhERGZUTbhvB9aY2SozawLuBbZMabMF+N1w+reAH6jeLiJSPzPW3MMa+v3ANoJDIb/m\n7jvN7CGg1923AP8IfMPM+oATBF8AIiJSJ1Ud5+7uW4GtU5Z9qmR6BPhgbbsmIiIXq/GO2xIRaQAK\ndxGRGFK4i4jEUN3uxGRm/cD+C3hJJ415UpQ+d+Np1M+uz12dFe4+47HkdQv3C2VmvdXcWipu9Lkb\nT6N+dn3u2lJZRkQkhhTuIiIxFKVw31zvDtSJPnfjadTPrs9dQ5GpuYuISPWiNHIXEZEqRSLczWy9\nme0xsz4ze7De/ZktZtZtZs+Y2S4z22lmD4TLF5nZ98zs9fB5Yb37OhvMLGlmL5nZd8P5VWb2fLjf\n/0944bpYMbMFZvakmb1mZrvN7F2NsL/N7I/Df+OvmtljZtYcx/1tZl8zs2Nm9mrJsrL71wJfCj//\nDjO76VK2fdmHe8lt/u4C1gEbzWxdfXs1a3LAJ919HXAr8Inwsz4IPO3ua4Cnw/k4egDYXTL/aeAL\n7n4tcJLgdo5x87fAv7r7zwPvJPj8sd7fZrYM+EOgx92vI7ggYfH2nHHb348C66csq7R/7wLWhI9N\nwCOXsuHLPtwpuc2fu2eB4m3+YsfdD7v7T8PpQYL/0ZcRfN5/Cpv9E/CB+vRw9pjZcuC/AF8N5w34\nVYLbNkIMP7eZdQD/ieCqqrh71t1P0QD7m+CihS3h/R9agcPEcH+7+48IrpRbqtL+3QB83QPPAQvM\nbOnFbjsK4V7uNn/L6tSXOWNmK4EbgeeBJe5+OFx1BFhSp27Npi8CfwYUwvkrgFPungvn47jfVwH9\nwP8Ky1FfNbN5xHx/u/sh4HPAWwShfhp4kfjv76JK+7emWReFcG84ZtYG/F/gj9z9TOm68CYosTrE\nycx+DTjm7i/Wuy9zLAXcBDzi7jcCQ0wpwcR0fy8kGKWuAq4C5nF+6aIhzOb+jUK4V3Obv9gwszRB\nsH/T3b8dLj5a/PMsfD5Wr/7NktuBe8xsH0HZ7VcJatELwj/bIZ77/SBw0N2fD+efJAj7uO/vO4E3\n3b3f3ceAbxP8G4j7/i6qtH9rmnVRCPdqbvMXC2Gd+R+B3e7++ZJVpbcx/F3g/81132aTu/+5uy93\n95UE+/cH7v7bwDMEt22EeH7uI8ABM3tHuOgOYBcx398E5Zhbzaw1/Ddf/Nyx3t8lKu3fLcBHw6Nm\nbgVOl5RvLpy7X/YP4G5gL/Az4C/q3Z9Z/JzvJvgTbQfwcvi4m6D+/DTwOvB9YFG9+zqL/w3eC3w3\nnF4NvAD0Ad8CMvXu3yx83huA3nCffwdY2Aj7G/gfwGvAq8A3gEwc9zfwGMHvCmMEf6ndV2n/AkZw\nZODPgFcIjia66G3rDFURkRiKQllGREQukMJdRCSGFO4iIjGkcBcRiSGFu4hIDCncRURiSOEuIhJD\nCncRkRj6//EIu1w3eV/GAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd9619fdd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "\n",
    "# Number of gradient descent steps, each over a batch_size amount of data.\n",
    "vocab_size = 40000\n",
    "\n",
    "# Uniform chance of guessing any word. \n",
    "loss_random_guess = np.log(float(vocab_size))\n",
    "print(\"Loss for uniformly random guessing is\", loss_random_guess)\n",
    "\n",
    "sent_length = [5, 10, 25]\n",
    "# Outputs correct target x percent of the time. \n",
    "pred_accuracy = np.arange(100)\n",
    "\n",
    "plt.plot(pred_accuracy, [1./p for p in pred_accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f2374029358>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt4AAAHVCAYAAADGu/wBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl4VOX9///nLNlnssyEAAlhlUUQBUIjm+wqbhX0oyCK\nClSxIghtkR8Vra2iCKIiKFhFrBUVLwQvtYL98gEUQRS1qHyQJQYIkCBk3zOZmfP7IzpKAQOY5Ewm\nr8d15dKZc8857zN3KS/v3Oe+LYZhGIiIiIiISL2yml2AiIiIiEhToOAtIiIiItIAFLxFRERERBqA\ngreIiIiISANQ8BYRERERaQAK3iIiIiIiDUDBW0RERESkASh4i4iIiIg0AAVvEREREZEGYDe7ABER\nERH59QzD4Pjx41RXV5tdSpMVFhZGs2bNsFgspzxuaWxbxmdnZ9fr+RMTE8nNza3Xa8i5Ud8EJ/VL\n8FLfBCf1S3Cqj35JTk6u0/PV5tixY3i9XsLCwhr0uvKT6upq7HY7SUlJpzyuqSYiIiIiIaC6ulqh\n22RhYWG/+BsHBW8RERERkQag4C0iIiIi0gAUvEVERETkV8vKymLgwIFn3H7fvn0MGTKEoUOHsn//\n/nqsLHgoeIuIiIhIg1u7di3XXHMNGzZsoF27drW2NwwDv9/fAJXVHy0nKCIiIhKCRo6MO+m9Sy/1\nMHlyxTkdf/vtolqv6fV6ueuuu/jmm2/o3LkzixcvZt++fTz44IOUlZXhcrlYtGgR33zzDc8//zw2\nm43NmzezZs0alixZwuuvvw7AzTffzKRJk8jKymL06NH06tWLr7/+mtdee42MjAzmzZuHx+Ohbdu2\nLFy4EIfDcdbfjxk04i0iIiIidSIjI4Px48ezZcsWnE4nL730ErNmzWLZsmWsX7+esWPH8uijjzJ8\n+HBuu+02Jk2axJo1a/jqq6944403WLt2Le+//z6vvvoq33zzDQCZmZmMHz+ezZs3Ex0dzVNPPcWq\nVav43//9Xy666CKWLl1q8l2fOY14i4iIiISg2kaof+3xU0lJSeHiiy8G4H/+5394+umn2b17Nzfc\ncAMAfr//lGtcf/rpp1xxxRXExMQAcNVVV7Ft2zYuv/xyUlNT6d27NwBffPEFe/fu5eqrrwZqllD8\n8VhjoOAtIiIiInXiv3dsdDgcdO7cmbVr157zOaOjowP/bhgGgwYN4vnnnz/n85lJU01EREREpE4c\nPnyY7du3A7B69WrS0tLIy8sLvFddXc3u3btP+lyfPn1Yu3Yt5eXllJWV8f7779OnT5+T2qWlpfHZ\nZ5+RmZkJQFlZGd9991093lHd0oi3iIiIiNSJ8847j5deeolp06bRqVMnfve73zFkyBDuv/9+iouL\n8fl83HnnnXTp0uWEz1144YWMGTOGESNGADUPV3bv3p2srKwT2iUmJvLMM89w1113UVVVBcCsWbPo\n0KFDw9zgr2QxDMMwu4izkZ2dXa/nT0xMJDc3t16vIedGfROc1C/BS30TnNQvwak++iU5OblOz1eb\nI0eOEB4e3qDXlJN5PB5SUlJOeeyMRrx37NjB8uXL8fv9DBs2jJEjR55wvLq6msWLF5OZmYnT6WTa\ntGkkJSWRkZFxwhycG264gfT09DM6p4iIiIhIKKk1ePv9fpYtW8bs2bNxu93MmjWL3r1706pVq0Cb\nDRs2EBMTw6JFi9iyZQsrVqxg+vTppKamMnfuXGw2GwUFBcyYMYO0tDQsFkut5xQRERERCSW1PlyZ\nkZFBixYtaN68OXa7nX79+gUmyP/o888/Z/DgwUDN5PidO3diGAYRERHYbDagZlT8xyddz+ScIiIi\nIiKhpNYR7/z8fNxud+C12+1m3759p21js9mIjo6mpKSE2NhY9u3bx5IlSzh+/DhTpkzBZrOd0TlF\nREREREJJva9q0rFjR5588kkOHz7Ms88+S48ePc7q8+vXr2f9+vUAzJ07l8TExPooM8But9f7NeTc\nqG+Ck/oleKlvgpP6JTipX6Qh1Bq8XS4XeXl5gdd5eXm4XK5TtnG73fh8PsrLy3E6nSe0adWqFZGR\nkRw6dOiMzvmj4cOHM3z48MDr+n4SXE+bBy/1TXBSvwQv9U1wUr+Ya82aKFq39pKWVn3C+6GwqokE\nv1rneHfo0IGcnByOHTuG1+tl69atJ23NmZaWxqZNmwDYtm0b3bp1w2KxcOzYMXw+HwDHjx8nOzub\nZs2andE5RUREROpSVpaNGTPiWLTIWXtjOWtFRUW89NJLZpfBwYMHGTFiBOnp6dxxxx14PJ5Ttlu4\ncCHp6en07duXDRs2BN6/99576dq1KwMHDqzz2moN3jabjQkTJjBnzhymT59O3759SU1NZeXKlXz+\n+ecADB06lNLSUqZMmcJ7773HzTffDMDu3buZMWMGM2bM4IknnmDixInExsae9pwiIiIi9cEw4L77\n4rHZYM6cQrPLCUlFRUW8/PLLZpfBww8/zKRJk/jss8+Ii4tjxYoVJ7XZs2cPa9asYfPmzbzxxhvM\nnDkzMFg8ZswY3njjjXqp7YzmePfq1YtevXqd8N7o0aMD/x4eHs4f/vCHkz43cODA0/7XwqnOKSIi\nIlIf3nwzis2bI5gzp5CUFL/Z5TSIkW+fvEfKpW0uZXLPyed0/O2Rb//i9R555BEOHDjAkCFDGDRo\nEImJibzzzjtUVVVx5ZVXMnPmTLKyshgzZgxpaWls376dHj16cNNNNzFv3jxyc3NZsmQJvXr1Yt68\neRw4cID9+/eTn5/PPffcw7hx42q9Z8Mw+Pjjj1m6dClQk1fnz5/P+PHjT2i3bt06Ro0aRUREBG3a\ntKFdu3Z8+eWX/OY3v6Fv374n7ZhZV7RlvIiIiIS08nILDz8cS3p6FbfeWm52OSFr9uzZ7N69m40b\nN7Jx40bee+89PvjgAwzDYNy4cXzyySekpKSwf/9+XnzxRRYuXMhll13G6tWree+991i3bh1PP/00\nr7zyCgC7du1i7dq1lJeXM2zYMC699FIcDgfXXHPNKa+/dOlSEhMTiY2NxW6vibjJyckcPXr0pLY5\nOTmkpaUFXp+uXV1T8BYREZGQFh1t8NJLBbhcPqy1TrINHbWNUP/a479k06ZNbNq0iaFDhwJQVlZG\nZmYmKSkptG7dmq5duwLQuXNnLrnkEiwWC+effz6HDh0KnGPEiBFERUURFRVF//79+fLLL7nyyivZ\nuHHjaa/788U7gpGCt4iIiISssjILMTEG6emnfsBO6odhGEydOpXbbrvthPezsrKIiIgIvLZarYHX\nVqs1MM8aCGy8+PPXpaWlvzji3alTJ4qLi/F6vdjtdrKzs2nRosVJbVu2bEl2dnbg9ena1bUm9N99\nIiIi0pQUF1sYPLgZf/97jNmlNAkOh4PS0lIAhgwZwuuvvx54nZOTw/Hjx8/qfOvWraOyspL8/Hy2\nbt1Kz549cTgcgaks//3TuXNnLBYL/fv359133wVg5cqVjBgx4qRzX3755axZs4aqqioOHjxIZmZm\ngzx7qOAtIiIiIemRR2I5etTGxRdrtLshuFwu0tPTGThwIB9++CHXXXcdV111FYMGDWLChAmBEH6m\nunbtyqhRo7jiiiv4wx/+cMYj0g888ABLly4lPT2dgoKCwGp769atY+7cuQB06dKFa6+9lgEDBjBm\nzBgef/xxbDYbAJMmTeLKK68kIyODiy666JSropwri2EYRp2drQH8/NcC9UEbGwQv9U1wUr8EL/VN\ncFK/NIwtW8K58cZE7rqrlAceKK61fShsoHPkyBHCw8Mb9Jr1Zd68ecTExDB58mSzSzlrHo+HlJSU\nUx7TiLeIiIiElPJyCzNmxNO2rZc//anE7HJEAvRwpYiIiISUbdvCyc628frreURFNapf7MsP7rvv\nPrNLqBcK3iIiIhJShg6t4pNPvqdly6axUY40HppqIiIiIiGhqgq2bq2Z46zQLcFIwVtERERCwjPP\nOLnhhkR27dIv9CU4KXiLiIhIo/d//2dn8WIH119fTteuXrPLETklBW8RERFp1Lxe+NOf4omP9/PQ\nQ0Vml9NkFRUV8dJLL5ldBgcPHmTEiBGkp6dzxx134PGceh33hQsXkp6eTt++fdmwYUPg/Q0bNtC3\nb1/S09N55plnAu8vW7aM9PR0kpKSznlregVvERERadSWLHHw9dfhPPpoES6XVjExS1FRES+//LLZ\nZfDwww8zadIkPvvsM+Li4k65Ac6ePXtYs2YNmzdv5o033mDmzJn4fD58Ph8zZ87k9ddf5+OPP2b1\n6tXs2bMHgPT0dFatWkVqauo516ZJUCIiItKoRUYaXHddOVddVWl2KUElbuTIk97zXHopFT9sSnO2\nx4vefvsXr/fII49w4MABhgwZwqBBg0hMTOSdd96hqqqKK6+8kpkzZ5KVlcWYMWNIS0tj+/bt9OjR\ng5tuuol58+aRm5vLkiVL6NWrF/PmzePAgQPs37+f/Px87rnnHsaNG1frPRuGwccff8zSpUsBGD16\nNPPnz2f8+PEntFu3bh2jRo0iIiKCNm3a0K5dO7788ksA2rVrR9u2bQEYNWoU69ato3PnznTv3r3W\n69dGwVtEREQatTvuKKNx7cMdmmbPns3u3bvZuHEjGzdu5L333uODDz7AMAzGjRvHJ598QkpKCvv3\n7+fFF19k4cKFXHbZZaxevZr33nuPdevW8fTTT/PKK68AsGvXLtauXUt5eTnDhg3j0ksvxeFwcM01\n15zy+kuXLiUxMZHY2Fjs9pqIm5yczNGjR09qm5OTQ1paWuD1z9v9fNfJli1bBgJ5XVDwFhERkUZp\nxYpo4uL8XH11JRaL2dUEn9pGqH/t8V+yadMmNm3axNChQwEoKysjMzOTlJQUWrduTdeuXQHo3Lkz\nl1xyCRaLhfPPP59Dhw4FzjFixAiioqKIioqif//+fPnll1x55ZVs3LjxtNc917nXDUXBW0RERBqd\nzEwbDz4YxyWXVHHVVQrewcYwDKZOncptt912wvtZWVlEREQEXlut1sBrq9WKz+cLHLP8V6daLBZK\nS0t/ccS7U6dOFBcX4/V6sdvtZGdn06JFi5PatmzZkuzs7MDrn7c7cuRI4P2cnBxatmx5prddKz1c\nKSIiIo2K31+zikl4uMHcuYUK3UHC4XBQWloKwJAhQ3j99dcDr3Nycjh+/PhZnW/dunVUVlaSn5/P\n1q1b6dmzJw6HIzCV5b9/OnfujMVioX///rz77rsArFy5khEjRpx07ssvv5w1a9ZQVVXFwYMHyczM\npFevXvTs2ZPMzEwOHjyIx+NhzZo1XH755b/ym/mJgreIiIg0Kq+8Es2nn0bw0ENFtGihHSqDhcvl\nIj09nYEDB/Lhhx9y3XXXcdVVVzFo0CAmTJgQCOFnqmvXrowaNYorrriCP/zhD6ccuT6VBx54gKVL\nl5Kenk5BQQE333wzUBPk586dC0CXLl249tprGTBgAGPGjOHxxx/HZrNht9uZO3cuo0ePpn///lx7\n7bV06dIFgBdeeIGLLrqI7OxsBg8ezPTp08/qfgAshtG4Hkf4+a8F6kNiYiK5ubn1eg05N+qb4KR+\nCV7qm+Ckfvl1srOtDBqURHq6h1dfza+z0e766Jfk5OQ6PV9tjhw5Qnh4eINes77MmzePmJgYJv+w\nwkpj4vF4TnhA8+c0x1tEREQajZYt/Tz4YDFDh2petzQ+Ct4iIiLSKHg8EB4O48aVm12K1LP77rvP\n7BLqheZ4i4iISNDLyrLRt29zNm2KqL2xSJBS8BYREZGg5vfDH/8YT2mphY4dvWaXE7TCwsKorq42\nu4wmrbq6mrCwsNMe11QTERERCWr//Gc0W7dGMG9eISkpvto/0EQ1a9aM48eP4/F4zC6lyQoLC6NZ\ns2anPa7gLSIiIkHr0CEbjzwSyyWXVDF2rOZ2/xKLxUJSUpLZZcgv0FQTERERCVqrVkVhtcITT2ij\nHGn8FLxFREQkaE2bVsoHHxynVStNMZHGT8FbREREgs7hwzb277dhsUDbtgrdEhoUvEVERCSo+P0w\nbVo811+fSGWl2dWI1B09XCkiIiJB5eWXY/jkkwgWLCggMtLsakTqjka8RUREJGjs329jzhwnQ4dW\nMnp0hdnliNQpBW8REREJCj4fTJ8eT3g4zJunVUwk9GiqiYiIiASFqioLrVv7GDu2nJYt/WaXI1Ln\nFLxFREQkKERHGzzzTKHZZYjUG001EREREVN5vTBjRhzffqvxQAltCt4iIiJiqiVLHLz2Wgx79yp4\nS2hT8BYRERHT7NplZ8ECJ9dcU8G112rRbgltCt4iIiJiCo8H7r03gfh4P48+WmR2OSL1Tr/TERER\nEVMsWxbDrl1hLF+eh8ulVUwk9Cl4i4iIiCluvbWcxEQ/l11WZXYpIg1CwVtEREQaVGUlGAbExBjc\ncIN2p5SmQ3O8RUREpEHNnRvLlVc2o6JCW1NK06LgLSIiIg1my5ZwXnjBQb9+HqKiDLPLEWlQCt4i\nIiLSIEpKLEyfHk+7dl7uv7/Y7HJEGpzmeIuIiEiD+Mtf4sjJsfH227lER2u0W5oejXiLiIhIvSst\ntfCf/4QxeXIpaWnVZpcjYgqNeIuIiEi9czgM1q49jlVDftKE6X/+IiIiUm8MA155JZrSUguRkRAe\nbnZFIuZR8BYREZF6s2pVFLNmxbNqVZTZpYiYTsFbRERE6sXhwzYeeCCOiy+uYty4crPLETGdgreI\niIjUOb8fpk2Lx++Hp58uxGYzuyIR8+nhShEREalzL7wQwyefRPDkkwW0bu0zuxyRoKDgLSIiInVu\n8OAq8vJKuPHGCrNLEQkammoiIiIidcbvr/ln585e/vznEiwWc+sRCSYK3iIiIlJn5s51MmVKPF6v\n2ZWIBB8FbxEREakTn30WznPPOYiMNLBrMqvISc7oj8WOHTtYvnw5fr+fYcOGMXLkyBOOV1dXs3jx\nYjIzM3E6nUybNo2kpCS+/vprVqxYgdfrxW63M27cOC644AIAHnroIQoKCgj/YSX92bNnExcXV8e3\nJyIiIg2hpMTClCnxtG7t4y9/KTa7HJGgVGvw9vv9LFu2jNmzZ+N2u5k1axa9e/emVatWgTYbNmwg\nJiaGRYsWsWXLFlasWMH06dNxOp3MnDkTl8tFVlYWc+bM4fnnnw98burUqXTo0KF+7kxEREQazAMP\nxJGdbWPNmlwcDsPsckSCUq1TTTIyMmjRogXNmzfHbrfTr18/tm/ffkKbzz//nMGDBwPQp08fdu7c\niWEYtGvXDpfLBUBqaioej4fq6uq6vwsRERExzeHDNv71r0juvbeU3r3197zI6dQ64p2fn4/b7Q68\ndrvd7Nu377RtbDYb0dHRlJSUEBsbG2jz6aef0r59e8LCwgLvPffcc1itVi6++GKuv/56LKd49Hn9\n+vWsX78egLlz55KYmHiWt3h27HZ7vV9Dzo36JjipX4KX+iY4hWK/JCbCF194SU2NICwswuxyzkko\n9osEnwZ59OHQoUOsWLGC+++/P/De1KlTcblcVFRUsGDBAj766CMGDRp00meHDx/O8OHDA69zc3Pr\ntdbExMR6v4acG/VNcFK/BC/1TXAKpX7x+2HjxgiGDq0iNhaKisyu6NzVR78kJyfX6fmk8at1qonL\n5SIvLy/wOi8vLzB95FRtfD4f5eXlOJ3OQPsnnniCyZMn06JFixM+AxAVFcWAAQPIyMj49XcjIiIi\nDeall2K49VY3mzY1zlFukYZWa/Du0KEDOTk5HDt2DK/Xy9atW+ndu/cJbdLS0ti0aRMA27Zto1u3\nblgsFsrKypg7dy5jx46lS5cugfY+n4/i4ponnr1eL1988QWpqal1eFsiIiJSn3bvtvPoo7Fcemkl\ngwdXmV2OSKNQ61QTm83GhAkTmDNnDn6/nyFDhpCamsrKlSvp0KEDvXv3ZujQoSxevJgpU6bgcDiY\nNm0aAOvWrePo0aOsWrWKVatWATXLBkZERDBnzhx8Ph9+v5/u3bufMJ1EREREgldVFdxzTwJOp58n\nnijU7pQiZ8hiGEajWvMnOzu7Xs8fSnPvQo36JjipX4KX+iY4hUK//PWvsfz97w5eeSWPYcNCY7Rb\nc7ylIWjnShERETkr3btXM3lySciEbpGGog1dRURE5Kxcd12F2SWINEoa8RYREZFaGQbcc088K1ZE\nm12KSKOl4C0iIiK1ev31aNasiaakRE9SipwrBW8RERH5Rd99Z+PBB2MZMKCKO+8sM7sckUZLwVtE\nREROy+OpWTowIgKefroAq5KDyDnTw5UiIiJyWv/+dyRffx3OCy/k07Kl3+xyRBo1BW8RERE5rauv\nruT9949z0UXVZpci0ujpF0YiIiJykvx8C7t21YzPKXSL1A0FbxERETmBYcCMGfGMGpVIYaFWMRGp\nKwreIiIicoJXX41m3boo/vCHEuLjDbPLEQkZCt4iIiISsHevnYceimXQoEruuENLB4rUJQVvERER\nAaCyEiZPTiA62uCppwq1dKBIHdOqJiIiIgLUzO3u1cvD8OGVNG+upQNF6pqCt4iIiAAQFQWPP15k\ndhkiIUu/RBIREWnijh61MnKkO7B8oIjUDwVvERGRJszng6lTE9i5M4zwcLOrEQlt+k9bERGRJmzJ\nEgdbtkSwYEEB553nNbsckZCmEW8REZEm6osvwpg3z8k111QwenSF2eWIhDwFbxERkSbq2WcdtGzp\n4/HHC7Fog0qReqepJiIiIk3Us88WkJ1tIy5Ou1OKNASNeIuIiDQxH38cTkmJhago6NDBZ3Y5Ik2G\ngreIiEgTsmePndtuc/HXv8aaXYpIk6PgLSIi0kRUVMDvf5+Aw2Fw330lZpcj0uRojreIiEgT8dBD\ncezZE8Zrr+WRlKQt4UUamka8RUREmoB3343k1VdjuPvuEgYNqjK7HJEmScFbRESkCTj//GrGjCnT\nFBMRE2mqiYiISAjz+cBqhfPO87FgQZHZ5Yg0aRrxFhERCWGPPBLL5Mnx+LRqoIjpFLxFRERC1L//\nHcHf/+7A5fJjs5ldjYgoeIuIiISgI0esTJ+ewAUXeJg9u9jsckQEBW8REZGQ4/XC5MkJVFfDkiUF\nREaaXZGIgIK3iIhIyNm9286uXWHMm1dE+/aa3C0SLLSqiYiISIi54AIvW7Yco1kzbZIjEkw04i0i\nIhIicnKsvPZaNIaBQrdIENKIt4iISAj4cV73N9+EMXhwJcnJCt4iwUbBW0REJATMn+/k008jWLSo\nQKFbJEhpqomIiEgjt2FDBIsXOxk7tozrrqswuxwROQ0FbxERkUasqMjC1KnxnH9+NX/7m7aEFwlm\nmmoiIiLSiMXFGTzySDEXXOAhKsrsakTklyh4i4iINFLHjllJSvIzcqSml4g0BppqIiIi0gj9+98R\n9O3bnG3bws0uRUTOkIK3iIhII5OVZWPatAQ6dqymRw+P2eWIyBlS8BYREWlEqqrgrrsSMAx4/vkC\nIiPNrkhEzpTmeIuIiDQif/tbHF99Fc6yZfm0aeMzuxwROQsa8RYREWkk/P6aHSrvvLOUESMqzS5H\nRM6SRrxFREQaCasVHn+8CL82phRplDTiLSIiEuTKyy1MmJDAzp0142VW/e0t0ijpj66IiEgQMwyY\nOTOOf/87kvx8m9nliMivoOAtIiISxP75z2hWr47mj38sYeDAKrPLEZFfQcFbREQkSH31VRh/+Usc\nQ4dWcu+9pWaXIyK/koK3iIhIkHrmGQfNmvlYuLBA87pFQoBWNREREQlSixcXkJNjw+UyzC5FROqA\n/vtZREQkyPzrX5GUlFiIioL27bVJjkioUPAWEREJIps2RTBpUgKLFjnMLkVE6piCt4iISJA4dMjG\n5MkJdOniZfp0PUwpEmoUvEVERIJAZSXceWcCfj+88EI+UVGa1y0Sas7o4codO3awfPly/H4/w4YN\nY+TIkSccr66uZvHixWRmZuJ0Opk2bRpJSUl8/fXXrFixAq/Xi91uZ9y4cVxwwQUAZGZm8uyzz+Lx\neOjZsyfjx4/HYrHU/R2KiIg0AnPmxPL11+EsX55Hu3aa1y0Simod8fb7/Sxbtow///nPPPXUU2zZ\nsoXDhw+f0GbDhg3ExMSwaNEirrrqKlasWAGA0+lk5syZLFiwgMmTJ7No0aLAZ1544QUmTZrEM888\nw9GjR9mxY0cd35qIiEjjcfvtZcyZU8hll2mTHJFQVWvwzsjIoEWLFjRv3hy73U6/fv3Yvn37CW0+\n//xzBg8eDECfPn3YuXMnhmHQrl07XC4XAKmpqXg8HqqrqykoKKCiooJOnTphsVgYOHDgSecUERFp\nCo4ft2IY0KGDj9tvLze7HBGpR7VONcnPz8ftdgdeu91u9u3bd9o2NpuN6OhoSkpKiI2NDbT59NNP\nad++PWFhYac8Z35+/imvv379etavXw/A3LlzSUxMPIvbO3t2u73eryHnRn0TnNQvwUt9E5x+3i95\neXDttWFce62f+fM1vcRM+vMiDaFBNtA5dOgQK1as4P777z/rzw4fPpzhw4cHXufm5tZlaSdJTEys\n92vIuVHfBCf1S/BS3wSnH/vF54NbbnGRkwOXX55Pbm612aU1afXx5yU5OblOzyeNX61TTVwuF3l5\neYHXeXl5gekjp2rj8/koLy/H6XQG2j/xxBNMnjyZFi1anPE5RUREQtm8eU4++iiSRx8tokcPhW6R\npqDW4N2hQwdycnI4duwYXq+XrVu30rt37xPapKWlsWnTJgC2bdtGt27dsFgslJWVMXfuXMaOHUuX\nLl0C7RMSEoiKimLv3r0YhsFHH3100jlFRERC1fvvR7J4sZObby7jpps0r1ukqah1qonNZmPChAnM\nmTMHv9/PkCFDSE1NZeXKlXTo0IHevXszdOhQFi9ezJQpU3A4HEybNg2AdevWcfToUVatWsWqVasA\nmD17NnFxcfzud7/jueeew+Px0KNHD3r27Fm/dyoiIhJEBgyo4uGHi8wuQ0QakMUwjEa1Qn92dna9\nnl9zIoOX+iY4qV+Cl/om+BgGNGtW0y+GAdq+Inhojrc0BO1cKSIi0gD8frj77gRefLHmr16FbpGm\nR8FbRERqfhF+AAAgAElEQVSkASxa5OCdd6IoKzO7EhExi4K3iIhIPVu/PoL5851cd105U6f6zS5H\nREyi4C0iIlKPMjNtTJmSQNeuXubNK9IUE5EmTMFbRESkHn3wQSQ2m8GyZflERTWq9QxEpI4peIuI\niNSj3/++jA0bjpOaqi3hRZo6BW8REZF68I9/RPPNN2EAJCVpXreIKHiLiIjUufXrI7j//jiWL48x\nuxQRCSIK3iIiInUoI8POPfck0K1bNXPmaGdKEfmJgreIiEgdKS62MGFCAuHhBi+9VKCHKUXkBHaz\nCxAREQkVixY5OHjQzsqVeaSk6GFKETmRgreIiEgd+eMfSxg4sIo+fTxmlyIiQUhTTURERH6lTz4J\np6jIQmQkXHKJQreInJqCt4iIyK/wf/9nZ9w4F7Nnx5ldiogEOQVvERGRc5Sba2X8eBdxcQYPPFBs\ndjkiEuQ0x1tEROQceDxw550J5OXZWLMmV5vkiEitFLxFRETOwWOPxfLppxE8+2wBF15YbXY5ItII\nKHiLiIicg9tvL6N1ay8jR1aYXYqINBKa4y0iInIWDh60YRjQpo2P8ePLzS5HRBoRBW8REZEzdOCA\njSuvbMbcuU6zSxGRRkjBW0RE5AwUF1u4/XYXAGPGaKRbRM6e5niLiIjUwueDyZMT2L/fzmuv5dGu\nnbaDF5Gzp+AtIiJSi0ceiWXDhkjmzi2kf3/tTCki50bBW0REpBZ9+ngICyth3DhNMRGRc6fgLSIi\nchrl5Raiow0uv7ySyy+vNLscEWnk9HCliIjIKWRl2RgwIIk1a6LMLkVEQoSCt4iIyH8pKalZwaSy\n0kL37prTLSJ1Q1NNREREfsbrhbvvTiAjw86KFXmcd55WMBGRuqHgLSIi8jN/+1vNCiaPP17IJZdo\ntFtE6o6mmoiIiPzAMCAsDH73u1JuuUUrmIhI3dKIt4iICOD3g9UKDzxQjGGYXY2IhCKNeIuISJOX\nkWFj2LBm7NxZMx5lsZhckIiEJI14i4hIk5afb+W229yUlFiIi9NQt4jUHwVvERFpsqqqYMKEBI4e\ntfHmm7mkpmoFExGpPwreIiLSJBkG/OlP8WzfHsGSJfmkpVWbXZKIhDjN8RYRkSapvNzCoUM2Zs4s\n5re/1XbwIlL/NOItIiJNUkyMwZtv5hEWZnYlItJUaMRbRESalM8+C2fcOBeFhRbCw7WCiYg0HI14\ni4hIk5GZaWP8eBcul19rdYtIg9OIt4iINAn5+VbGjXNjtRr88595JCQoeYtIw9KIt4iIhLzKyppl\nA3NybKxcmUvbtlo2UEQanka8RUQk5B06ZGf/fjsLFxbwm99o2UARMYdGvEVEJOR17Ohly5ZjOBya\nXiIi5tGIt4iIhKzXX49m3jwnhoFCt4iYTsFbRERC0saNEcycGcdXX4Xh05RuEQkCCt4iIhJydu60\nM2lSAl26eHn++QLsmlgpIkFAwVtERELKkSM2br3VTVycn1deydMUExEJGhoDEBGRkLJtWzhVVRbe\neiuPFi38ZpcjIhKg4C0iIiHl+usrGDq0UhvkiEjQ0VQTERFp9AwDZs2K48MPIwAUukUkKCl4i4hI\nozd3rpNXXonhq6/CzC5FROS0FLxFRKRRe/nlaBYvdnLLLWVMmVJqdjkiIqel4C0iIo3W2rWRzJ4d\nx2WXVTBnThEWi9kViYicnoK3iIg0Wv/6VyQ9e1bz3HOFWqtbRIKe/m9KREQarYULCykrsxAVpYcp\nRST4acRbREQalaNHrdx6q4vsbCs2G8TGKnSLSOOgEW8REWk0ioos3HKLm6wsG3l5NpKTtUGOiDQe\nZxS8d+zYwfLly/H7/QwbNoyRI0eecLy6uprFixeTmZmJ0+lk2rRpJCUlUVJSwpNPPklGRgaDBw9m\n4sSJgc889NBDFBQUEB4eDsDs2bOJi4urw1sTEZFQUlkJEye6yMiw88or+XTvXm12SSIiZ6XW4O33\n+1m2bBmzZ8/G7XYza9YsevfuTatWrQJtNmzYQExMDIsWLWLLli2sWLGC6dOnExYWxujRo8nKyuLQ\noUMnnXvq1Kl06NChbu9IRERCjs8HU6cm8MknESxeXMDAgVVmlyQictZqneOdkZFBixYtaN68OXa7\nnX79+rF9+/YT2nz++ecMHjwYgD59+rBz504MwyAyMpIuXboERrVFRETORV6elZ07w3jwwSJGjaow\nuxwRkXNS64h3fn4+brc78NrtdrNv377TtrHZbERHR1NSUkJsbOwvnvu5557DarVy8cUXc/3112M5\nxQKs69evZ/369QDMnTuXxMTE2u/qV7Db7fV+DTk36pvgpH4JXqHSN4YBiYnwxRd+nM4oIMrskn6V\nUOmXUKN+kYZg2sOVU6dOxeVyUVFRwYIFC/joo48YNGjQSe2GDx/O8OHDA69zc3Prta7ExMR6v4ac\nG/VNcFK/BK9Q6JvXXotm+/Zw5s+vWae7KgRmmIRCv4Si+uiX5OTkOj2fNH61TjVxuVzk5eUFXufl\n5eFyuU7bxufzUV5ejtPprPW8AFFRUQwYMICMjIyzLl5ERELX2rWRzJwZx/HjVvxavEREQkCtwbtD\nhw7k5ORw7NgxvF4vW7dupXfv3ie0SUtLY9OmTQBs27aNbt26nXLayI98Ph/FxcUAeL1evvjiC1JT\nU3/FbYiISCjZujWcyZMT6NGjmr//vQA9KiQioaDWqSY2m40JEyYwZ84c/H4/Q4YMITU1lZUrV9Kh\nQwd69+7N0KFDWbx4MVOmTMHhcDBt2rTA5ydPnkx5eTler5ft27cze/ZsEhMTmTNnDj6fD7/fT/fu\n3U+YTiIiIk3Xzp12xo930aaNl1deySM6WhvkiEhosBiG0aj+Hy07O7tez6+5d8FLfROc1C/Bq7H2\nzb//HcHf/hbHm2/mhuQGOY21X0Kd5nhLQ9DOlSIiEhS8XrDb4bLLqhgy5BhhYWZXJCJSt2qd4y0i\nIlLfCgstXH11IqtX1ywVqNAtIqFIwVtERExVXm7h1lvd7NkTRmKiz+xyRETqjaaaiIiIaTweuOOO\nBP7znzCef76AgQM9ZpckIlJvFLxFRMQUPh9MnZrApk2RPPFEIVdeWWl2SSIi9UpTTURExBQWCyQm\n+njggSJuuqnc7HJEROqdRrxFRKTBFRZaiI83ePjhYn5hvzURkZCiEW8REWlQixc7GD48iZwcq0K3\niDQpCt4iItJgXn45mscei6VPnyqSkkJvcxwRkV+i4C0iIg3irbeiuP/+eC69tJKnnirEZjO7IhGR\nhqXgLSIi9W7TpgimT4+nX78qli7N1wY5ItIkKXiLiEi96969mtGjy1m+PJ/ISLOrERExh4K3iIjU\nm9277Xg84Hb7mT+/CIfDMLskERHTKHiLiEi9+OqrMEaOTOSvf40zuxQRkaCg4C0iInXu22/tjB3r\nJj7ez+TJJWaXIyISFBS8RUSkTn33nY2bbnITGWmwcmUeyclaNlBEBLRzpYiI1CGvF8aPd+H3w6pV\nebRp4zO7JBGRoKHgLSIidcZuh3nzinA6/Zx3ntfsckREgoqmmoiIyK/2/fdWVq+OAqBPHw/duil0\ni4j8N414i4jIr5KXZ2XMGDdHjti45JIqmjXTnG4RkVNR8BYRkXNWUGBh9Gg3WVk2Xn01X6FbROQX\nKHiLiMg5KS62MHasm8xMOy+/nE/fvh6zSxIRCWoK3iIick7eeSeKb78N48UX8xk4sMrsckREgp6C\nt4iInJObby7n4os9dOyoBylFRM6EVjUREZEzVl5u4Y47Eti1y47FgkK3iMhZUPAWEZEzUlFh4dZb\nXaxbF0lmpn5hKiJythS8RUSkVhUVcPvtLj79NJxnnink6qsrzS5JRKTR0ZCFiIj8oooKmDDBxZYt\n4Tz1VCGjRlWYXZKISKOkEW8REflFXq+FigoLCxYUcsMNCt0iIudKI94iInJKFRVgGBacToO33srD\nZjO7IhGRxk3BW0RETlJRARMnuvD5LLz+ukK3iEhd0FQTERE5wY+h+6OPIrjuunKs+ptCQkBhVSFb\ns7fywjcv8H3592aXI02URrxFRCTgxwcpN2+OYMGCQkaP1pxuaVz8hh+f4SPMGsbOvJ08+cWT/F/e\n/3G49HCgTZvYNlzW5jITq5SmSsFbREQCpk5NUOiWRqPKV8XO3J18m/8tu/J3sStvF9/mf8vD/R7m\nxk43ApBRmEFa8zRuPf9Wurm70dXdlaToJJMrl6ZKwVtERAImTy7liisque46hW4JHn7DT1ZJFrvz\nd/Nt/rf0aNaDIalDOFJ6hN++81sAHGEOznedz/Udr6d9XHsALnBfwEc3fmRm6SInUPAWEWniSkst\nrFsXyf/8TwU9elTTo0e12SVJE5ZfmU+Ft4IURwrl1eXc+P6N7MnfQ7m3HAALFib3mMyQ1CG0jW3L\nskuX0dXVlVRnKhaLxeTqRX6ZgreISBNWUmLh5pvd7NgRRo8eHs47z2d2SdLEvLn3Tb7N/5Y9+XvY\nXbCb78u/59oO1/Lc0OeIDoumWVQzenbuyfmu8znfdT6dEzoTHRYNgNViZUTbESbfgciZU/AWEWmi\nioos3HKLm6+/DmPJkgKFbqkXVb4qMgoz2Fuwl90Fu9lbsJdmUc2Yd8k8ABb+ZyFHy47SKaETA1MG\n0sXVhbTmaYHPL79suVmli9Q5BW8RkSYoP9/C2LFudu8O4/nnCxgxotLskqSRq/JVkVmUyd6CvZR4\nSrjl/FsAGPXOKL7K/QoAu8VO+7j2tHK0Cnxu9TWrSYxMxGbVYvES+hS8RUSaoPXrI9m7N4xly/IZ\nNqzK7HKkEanwVnCo5BCdEjoBMP/z+byT+Q4Hig/gN/wAuCJd3NzlZiwWC3dfdDc+w0fnhM60j2tP\nuC38hPM1j27e4PcgYhYFbxGRJsQwwGKBG2+soF8/D61aaXqJ/LKPjnzEpkOb2Fe4j30F+zhcehi7\n1U7G+AzsVjvhtnC6uLrw2/a/pVNCJ86LP48OcR0CDzpe3f5qk+9AJHgoeIuINBFHjtiYODGBRx8t\nolevaoVuASC3IpeduTvZV7iPjMIMviv6jozCDD6+8WMc4Q62ZG/h5V0v0z6uPb2a92J059F0jO8Y\nGN2+t+e9Jt+BSOOh4C0i0gQcOGBj9Gg3xcVWDMPsaqShlXpKySzK5Lui72p+Cr/job4P0Ty6OSv3\nrOTR7Y8CEB8Rz3nx5zEsdRgV3goc4Q6m9ZzGfWn3aQ62SB1Q8BYRCXH79tkZM8ZNZaWFlSvzuPBC\nrdMdirx+L4dKDpFZlElmUSZXtbuKZEcyb+59k+kfTg+0s1qstHa25njFcZpHN+e3HX5L7+a9OS/+\nPFyRrpPWwo6yRzX0rYiELAVvEZEQlpFh47rr3Njt8NZbuXTp4jW7JPkV/Iafo2VH2V+8n/Piz6N5\ndHO25Wzjvs33cbD4IF7jp/5NdaaS7EimR7Me/H+/+f9oH9eeDnEdaBvblkh75AntUp2pZtyOSJOj\n4C0iEsJatfIxdGgV995bQvv2mtPdGBiGwbGKY4Rbw0mITCCzKJPHPnuM/cX7OVB8gApvBQBPDnyS\n0Z1H44p00TmhM1e0u4L2ce0DAdsV6QKgU0KnwAokImIuBW8RkRC0fXsYfftCZCQsXFhodjnyX/yG\nH4/PQ6Q9kvzKfJZ+vZT9xfvZX/RTuP5Ln79wZ/c7sVvs7C7YTbvYdvRP7h8I193c3YCaYP3CpS+Y\nfEdBwuvFWlCANTcXa14ent/8BiIiiNi4kci1a7Hm5WHLzaX4vvvw9O9vdrXSBCl4i4iEmPXrI5g0\nycWNN/p57DGzq2m6DMPAYrFQ4a3g9d2vc6DkAAeLD3K47DD7C/dz90V386e0PwHw92/+Tmtna9rG\ntqV/cn/axbajb8u+ALSObc3mGzebeSums37/Pfa9e7Hm52PLy8P6w0/Jfffhd7mIfvllnE88gbWw\nEMvPnh7+futWfG3aYN+7l8gPPsCfmIjf7TbxTqSpU/AWEQkhb78dxb33xtOtW7VCdwPx+X28t/89\nDhYfJKskiwPFB8gqyeKKtlfw175/xWqx8uAnDxJpj6RtbFs6J3ZmcMpgLm5xMQAJEQl8N/670F81\nxDCwlJZizc+vCc75+XjS0zFiYwnfto2oVatqRqR/djz37bfxdupE1LvvEveXv/x0KosFf0ICZRMn\n4ne58LVvT+U11+B3u/H9EK79bjf+pCQAyiZNomzSJLPuXCRAwVtEJES88ko0f/5zHH36eFi+PJ/E\nRDe5uWZXFRo2HNoQCNZZxVlklWTxmxa/4dH+j2K1WJmxeQZl1WUkRSXROrY1F7e4mB7NegAQYYtg\nxy07cEe6sVgsJCYmkvuzjrFYLNgsjSx0GwaWsjKMiAgIC8N2+DDh27fXhOqf/RTPnImvfXuiVq0i\nfsYMLB7PCac5/u67VPfqhe3IESI3bMDvcuF3u/FceCF+txsjOhqAyiuuoLpr18CItT8+Hmw/fWdV\nAwdSNXBgg34FIudCwVtEJAQUFVl48kknw4ZVsXRpPlFaAe6sbM3eSmZRJodKD3Go5BBZJVl0jO/I\nU4OeAmDG5hkcLTtKpC2S1s7WtI5tTYe4DkBNcF43ah0tolsQHRZ9yvMnRiU22L2cC0tFBdYjR2rm\nR//sp/Lqq/G1akX41q04n3wSa2FhTaguKMDi8ZC7ahWevn0J//xzEu65B/hpNNrvcmEtKsIHeDt3\npvTOOwPv+10u/AkJeDt3BqDi+uupuP7609bnS0nBl5LSEF+FSL1S8BYRacR+nM4aF2fw9tu5pKT4\nCAszt6Zg9GnOp+wr3Meh0kMcKTnCodJDJEUn8cLwmocSZ2+dzZ6CPdgtdlIcKTVL7Dl+WmLvtRGv\n4Yp0kRiVeNI61wDt49o32L2cls+Hpbi4Jhz/8FPdrRv+pCTse/YQ/eqrNe8XFAT+Wfj443gGDCBi\n40Zcd9xx0im97dvja9UKLBbw+fC2aYO/Z8+aAJ2QUHMMqBwyhO8//BC/y4URF3fCaDRAdffuVHfv\n3iBfg0gwU/AWEWmkfD7485/jcDoN7r+/mLZtm+5ygVuyt/Bt/rccKT3C4dLDHCk5QnRYNKuuXgXA\nY9sfY/v327Fb7CQ7kklxpNDG2Sbw+eeGPocjzEGLmBbYrSf/1djZ1bnB7oWKCqxFRTWjxUlJGAkJ\nWI8cIeq992reLyzE8kOwLp0yBU/fvkR8+CGum28+4cFCgPylS6m85hqsx44RvWpVTWCOj68ZbW7T\nBsPpBMDTowcFixYFAvWPP4HjffuSt2bNaUs24uLwxcXV33ciEiIUvEVEGqGqKpgyJYF//SuKe+4p\nMbuceuHxeQi3hQOw6dAm/nP8P2SXZnOk9AjZZdkYGHx4w4dAzaog67PWE2mLJMWRQitHKzomdAyc\na8HABUTaI2kR3eKUDzF2cXWpu8INA0tlJZbCQgyHA8PpxJqXR8SGDViLirBVVxObk4O1sJDysWPx\n9OlD2H/+g2vixJpQXVUVOFXBM89Qcf312I8cIe5vf8OwWvHHxWHEx+OPjw/Mmfa2a0fptGk1oTo+\nHn9cXE24Pu+8mu/ykks4+u23py3Zn5xMxXXX1d13ICKnpOAtItLIlJVZmDjRxebNETz4YBGTJpWZ\nXdJZ8/g8HK84ToqjZt7u2v1r2ZK9heyy7Jqf0myqfFXsvm03FouFtzLeYnXGappFNSPFkULH+I60\njm0dWLLv0f6P8uTAJ0+55TlAh/gOZ1dgZSXW4mKMqCgMpxNLYSERH36ItbgYa1FRzZSOoiIqRo7E\n07cv9t27cd1xR+B9S3U1AIXz51M+diy2Q4dImDYtcPpopxN/XByVw4cD4He7qRw2DCMuriY0/xCe\nq9PSar6vHj3I2bWrZgTaaj2pXF/r1pT86U9nd48i0uAUvEVEGhGfD8aMcfPVV2E89VQBN95YYXZJ\nJ6nwVpBTlsPRsqP0bdm3Jjjve4v3979Pdlk2OWU5HK84jgUL+yfuJ8waxsfZH7M6YzUtY1qSHJPM\nhYkXkhyTjM/wYbfYebjfw8y/ZP4JW53/3I8BHqiZ61xSgrWkBL/DgZGQgKWoiMj/9/9q3i8qwlpS\ngqWkhMqrr6Zq4EBs332H+/bba44XFwdGnQsfeYTy8eOx5eTguvvuwCWMsDD8cXF4evWCvn0xnE6q\nL7gAf2ws/vh4jNhY/LGxeNLTAaju0oXvt2zBHxeHu107cgtP3NTI17o1RfPnn/5LDQ/HCA8/xx4R\nkWCh4C0i0ojYbHDTTeVMmeLjssuqav9AHfIbfvIq8vi+/PuaYF1+lBs73UiELYJ/fvtP/rHrH+SU\n5VBY9VOo/GbcN7giXWSXZXOg+AAtY1pygfsCWsa0pGVMS/yGH4C/9v0rc/o9UjNF44dQbC0txXf0\nGP7kZBIqLUSvWlEzolxSUrMedHExFddeS+UVV2DLyiJx1Kiaz5X99BuAogceoOyuu7Dm5ZFw772B\n943ISPyxsYEH/gyHo2a5uri4QGj2x8bi6dMHqHnI8NjGjfhjYzFiYzGiomoeOPyBLyWFgiVLTv/l\nRUbia9u25t/t+qtXpKk6oz/9O3bsYPny5fj9foYNG8bIkSNPOF5dXc3ixYvJzMzE6XQybdo0kpKS\nKCkp4cknnyQjI4PBgwczceLEwGcyMzN59tln8Xg89OzZk/Hjx5/y14MiIgJ799rJybExaFAVY8eW\n1/n5Sz2lHC49zPfl3//0U/Y9f0z7IwmRCSz9eimPffYYXsN7wucGpgykTWwbHNYoelc1IzWsIyl2\nJy39DpJ8UcRnZMEFLqa2u5X73y2sCdWlpVhL/oOldDMVN0L5TTcRkfM9SX37YvGd+IBo8YwZlE6b\nhqW0lLgHHwTA/8P0D7/TiXXAgJr3nE4qBw+umVP9s+Bc3aNmLW1fairfb95cM5XD6YT/Gj32N29O\nwfPPn/4LiojA26nTr/2aRaSJqzV4+/1+li1bxuzZs3G73cyaNYvevXvT6oclhAA2bNhATEwMixYt\nYsuWLaxYsYLp06cTFhbG6NGjycrK4tChQyec94UXXmDSpEl07NiRxx57jB07dtCzZ8+6v0MRkUZu\n+/Ywbr/dTVycnw8/PHbGywX6/D58fh82q439RfvZlrON78u/53jZUUrycygr/J6/Dn6cVm0u4s1v\n/slnrz2C0wPOKnB4oLMvAu/o8+Hqm0kLb8+Od1JwVkFMlZ/ISi/h5VWUla2hfNo0rk8YyOQZ955U\nQ8lUPyUX9MDi8xH9j39gOJ0YMTH4f/in8cPorz8+ntLf/z4QqI3YWPwOB96ONQ9I+lu0IGfnTgyH\ng1N9AUZCAkULFpz+ywgLw9c+CJb8E5EmrdbgnZGRQYsWLWjevDkA/fr1Y/v27ScE788//5wbbrgB\ngD59+vDSSy9hGAaRkZF06dKFo0ePnnDOgoICKioq6PTD6MHAgQPZvn27greIyM8ZBv/+IJzfT3aT\n3Lyat+Z+RtTOYsqLjlFacJSyomM069aXyN8M4D8Ht1L92ANQWgJlZdjKKwivrKLl6LtJvPt+vsr4\nkHFj7sfhgeifDVpnTfgnPHwRQ5v1ZfYb/11AFSVdsyi5GtJT++P2xGI4HRgtHfgdDjwxMfi6dgVq\ngnPBggU1I86OmuOGw4Hvhy27/S4XRzMyTn+rMTGUzJp1+u/CZsNISDjHL1JEJDjUGrzz8/Nxu92B\n1263m3379p22jc1mIzo6mpKSEmJjY8/4nPn5+ed0AyIiQcHvx5aTg6WiAkt5ec1PRQW+Fi3wnn8+\nVFcT8+KLNTsE/ni8vJyq/v2puOEGKksLaDZyFEZZKdaKCuzlldgrPBRG3ELnzi8w+W8r6HHtxJMu\nu2f0biJ/M4D8qgJufH83FRE2qqLC8EZG4He68VLzMOKQzldjXPM5njg3XmdczWizw0FUt25UA21b\nXcjxtWvxR0cHwrMRHR1YQcOIiSF33brT3394OBVjxtTHNysiEjKC/gmP9evXs379egDmzp1LYmL9\nbrtrt9vr/RpybtQ3wSmo+6W6Gvx+iIioWVv522+hogLKy2t+KiogNRUjLQ18PqwLFmD58dgPx41L\nLsF/++1QXY198GAoK6tpU1EBZWX4b7sN31NPgcdDeGrqSSVUTLwd23PPcyA3g+RHHgGgKtxGRYSV\nsjAobx5Om8Tf837+Nmy+fZQ5ocwNpeFQHmbhMK3537chs+wCnp02gMg4N9Hu5sS6kolzJdOpywDi\nExMZnXgblN9GNPDjpuV2ux2vt2Z4OzExEV45aUj7RP9/e3ceH1V973/8dc6ZJZM9M2ENWAiLyqIo\nQQEFWV1QEXdFvSpKQeoC2t6LXHrrraJWRVEBsbeI1tpWaYv3anEpUqQl+hAE6lqRH7ZKWUIy2Wef\nOb8/JgxGNhfITJL38/GYRx6T75kzn5kvZ/LmO9/zPWPGHKl3Xg4ho4+Zdkz9Ii3hsMHb6/VSVVWV\nul9VVYXX6z3gNj6fj3g8TiAQIK/palffdp97jRs3jnFN65wCVFZWHq7k76S4uPioP4d8O+qbzPRN\n+sVobEwu0xYMYoTDGKEQdl4e8e7dwbbJeu01jFAIQqHk6hahELE+fQiPHQu2TcGdd6Z+v/cWOuMM\nGm+6CWw7eXJeMLivPRaj8aqrqH3gAbBtuh5gOlvj5MnJZdxsmy5NJ+/Z2dnJ9Zs9HoJeL/WVlYSi\nQQo8DkL5BQSdBTQ4beoccfJ7++hYWckHlR/wl8kl7DEaqKCRWitGoxOmn13KmZWVfLjzEwbfCUZ2\nNr6cjviyfBR7ipl+wiXkVFbSI6uUt36xGK+7GG+kI6d29eGhEMs0CYcrKXF048IfPb9f/TEO/rmo\nYyYzqV8y09Hol65dux7R/Unrd9jg3atXL3bu3ElFRQVer5fy8nJuvfXWZtsMHjyYNWvW0LdvX95+\n+2369+9/yBVKioqK8Hg8bNmyhT59+rB27VrOPvvs7/5qRGR/tp1a9sysrEwG33AYIxLBiERI5Oen\nltTEwqAAACAASURBVDnLeu21ZOjd2x4OEystJTx6NAD599yDEQwmH990M8eMgaYpBsXnn5+cQtG0\nD8JhQuedR+199wHQuV8/jFjzVTECl11GzSOPgGFQNG3a/u2XXpoM3oZB1p/+hG2akJWFnZWF7fHs\nWwXDMIgMH47tdCbbmtr3LheHYeB/8kliLicNjji1ZoQaM4K323F0AHYGdvGz30xlT6IOf7gaf8iP\nP+Rn1sl9uRD4uPrvnDf27f3e3vtPuIRrgGxnNm+NPQ6fx0eXrGIGeHx4s7wc1zm5HN3gToPZPH0r\nHofngN3UMbsjZ3a9gJtvLmLrVgevvlqJx2MfcFsREWmdDhu8LctiypQpzJs3j0QiwejRo+nevTvP\nP/88vXr1oqysjDFjxrBw4UJuueUWcnNzmfmlq3P94Ac/IBAIEIvFWL9+PXPnzqVbt27ceOONLF68\nmEgkwqBBg3RipbQ+iURyKoNtQ1ZyHq25YwdGNJq8al3Tz0ReHvGePQFwr1mTDKSRSGqbeLduRE47\nDYCcJ55IjtjubY9EiA4YQPDyywEovPnm5KhxUyg2olHCI0akrljXccSIfaPKTcE6OHEiNY8/DkCn\nU05pdjlqgMBFF6Xai2bMSI44f6V9b/D2vPACRjyeDLVuN7bLhdG/f2rbeMeOYBjJC300hd9I05X3\nAOrmzk2uYuF2Jx/vdhP/3vdS7XtWrky27d1/0z722v3uu/vefjtBbbgWh+kgD6iP1LNixghqwjVU\nh6qTP8M7mFjan/HA1pqtTKz6d2ojtc1e311FdzGVE2iMNrJky9N4s7wUZRXhzfLSz9cPb1by27jS\nglIWjV6E1+PF6/bizUre9l7QpbSglF+e/cuD/nNxmA4c5sE/cv1+g+uv9/Huu07uuqtOoVtEpA0y\nbNtuVZ/uO3bsOKr711eA35BtJwOoZQFgNDQkL60XjydHIuNxbJcLu2kqkfX//l9yRDMeh0QCIxYj\nUVCQCqaut97CiESgaRsjHifeqRPRk0+muLiYxiefTAbDWCy5/1iMWM+ehMePByD30UeTI65NbcRi\nyeDaNCJbcMcdyYtrxOPJYBuLETnlFBqavsXxXXwxZl1ds/bQ2LHUNc3N7XTSScn2aDQ10hqYNIma\nRYsA6NynD2ag+RrLgQsuoGbx4q/X3rcvZmMjtsOB7XSC00nw/POTUyWA4gkTMKLRVOjF6SQ8YgQN\nN9+cfH3//u/JnTqdyffd5SI6YACh888HIPu558A0U2222028a1diAwYA4Pjoo32PbXoO2+MBz4FH\naeG7HTMJO0HcjuM0nYRiITbs3kBtpJbacPJWE6nhtK6nMbJkJLsadzHl9SnUhGtS29jY3DnkTm4e\ndDNf1H/B0N8OTe0735VPobuQmwfdzFXHXYU/5Ofhdx+mKKuIIndR6mffor50ze3K3o/CdFxPYPt2\ni6uu8vLFFw4ee6ya884LHf5BX4M+zzKT+iUzaaqJtISMP7mypVmzZ1NQXd3sd9EBAwhceSUA+f/9\n38ng96X/r0QHDiRw1VXJ9p/8JPlVPCS3sW2iJ55I4JprAJJzVAOBVBtA9KSTaJwyBYDCWbOat9s2\nkbIyGqdPB6Bo+vR97YkEJBJETj2VhqZvGbzXXIPR2JgMtU3t4REjqP+P/wDAd+GFmPX1qceSSBAe\nNYq6n/4UgA6jRyeDZSKRCsehs85KrY/b6cQTMWtrk/tvCp7B88+nesmSZHtZWXL/X/Ll9g4TJmA2\nNDRvnzgxdcU37/XXH/LxBf/5nwdsTwXvphFj2+FIXh3O4SAYDqeCt+v99yEcTrZZFrbTua+/gITP\nRyI/P/VY2+Eg1rt3qj1w2WXJEV+HIxV8o8cem2qvvffeZN+4XMkpD04n8S998Fb97nf7gq/TCS4X\niZycVPuu999PrlHctJLEV1WuXHnA36eevymgH8zef6cHE2taGu7rStgJIvEIAJF4hI0VG6mL1CVv\n4TpqI7UM7jiYkd1GUh2qZtob05q11UXqmHnSTO4YfAe1kVouX3l5s/1bhoXH8jCyZCQehwdvlpfS\nglIK3YUUuAsoyirilE7JS3J3zunMm5e+SZG7iAJ3wX6jy94sL/ecds9BX0s6L+A1c2Yhe/ZY/PrX\nVQwdGklbHSIicnQpeH+FuWIFWbXNv4omFIKm4J31+usYe4Pf3j/U8Tg0BZqs1auTo7572wwjNRoM\n4HrnnX3tTbdEQUGq3fHJJ8ngvLfdNIkfc8y++vbsSbZbVmqbZlMH9j6f04ltGNimmRyxbBIvKSER\nCqX2/dX9R4YPTwbTpjYsi8jeObKQ/A9ENJpqty2r2dXc6mbPTo4UN7VhWcT2XiYZqJk/f98IuWVh\nm2bzYPrssxiQfKzDgW1Z2IWFqfY9b7yBbRjJcGpZ+0aGm+z6+ONml3H+qj2vv37QNoDqn//8kO2H\nXGcYCDatZ38w0RNPPGQ7bveh248Q27YJxUM0RBpwmA6KsoqIJqK88fkb1EfqaYg2UBepoyHSQFmn\nMs7qcRYNkQauevUq6iP11EXqUtv9cOgPmTVwFsFYkItfvni/55pxwgxGdhuJ23ITiUfo6OlIn8I+\nFLgKyHPlMbRLcpTal+Vj+bnLyXfnU+gqpNBdSI4zJxWIC9wF/OqcXx30NTlNJ70Lex+0PRPtnX4/\nf34N4bBB376xwz9IRERaLU01+Qp9BZi52nvf2LadDMPRBgLRAA3RBhqjjXTwdOBY77FEE1H+5/3/\noSHaQEOkIfkz2sDIkpFcffzVBGNBRi0fRUO0gfpIPXE7+Y3FDf1v4KfDf0ooFqLXsl7NntNtublx\nwI3MOWUO0USUq165inxXPnmuPPJceeS78hnfdzwn5p1Iwk6wbse6VKAucCd/Os2veZnFdubZZ7NZ\nv97FggU1B/uC4ztr78dMplK/ZCZNNZGWoBFvkSNo70hyMBYkEA0QjAXJdmZTkluCbdu8/NnLBGIB\ngtEgjdFGArEA/Xz9OLfnuSTsBDf86QYC0QCNscbkz2gj55eez9xT5xK34/T75f5TQa4+7mp+NuJn\nWIbFvHfmYWCQ68wlx5VDrjOX/r7kyY9ZVhZDuwwlz5lHriuXXGcuua597W7LzasXvkqeK49cZy55\nrjzc1r4ReKfp5IVzX9jv+ff+sTINkxElI47SO9t2JBLws5/lsXBhHmPHhohEUufmiohIG6fgLe3K\n7sBuQrEQ4XiYUCwZkIuykifYAbyw5QUCsUCqLRQP0c/bjwt6XQDA1FVTCUaDBGP7buO/N545p8zB\ntm16LO1BzG4+XeDyvpfz8BkPA/CD1T9IjTQDGBhMPm4y5/Y8F9Mw2dW4C5flIt+ZT5fsLngcnlRt\nDtPBT4f9FI/DQ44zhxxnMlh3zUmOqJiGyafXfYrH4TngfGXDMHh01KMHfW8Mw2Bg8cCDtst3FwrB\nHXcU8uKL2VxzTSP33FOLQ5/CIiLthj7y5aiIJ+JEEhEi8QimYZLnSl5QaWvNVsLxMNFElEg82b53\n2TaAP2z9A8FYkEg8QjgeJhKP0KewD+f0PAeAH5f/mIZoQ6o9HA9zaudTuXlQclWPs/5wFnWRulRb\nKBbi/NLzWTBqAQDDfjuMcLz5cnoX9b6Ix0cnl9O78693EorvW1HCaTq5pM8lqeD9r/p/YRgGWVYW\nPo+PLEcWJbklQDK4zjx5Jk7TSbYjm2xnNh6Hh575PVPtqy5eRZaVRY4zh2xnNllWVrOQ/MqFrxzy\nfb1hwP6XDP+ybGf2IdslfWwbpkzx8uabWcyZU8eMGQ2HOh1BRETaIAXvr6gOVlMdar6qidN0kuvK\nBcAf8mPbNjZ26qfbclPgTp4gubNxJwk70Wwbj8NDh+wOAGyr3UY8ESdhJ0iQIGEnyHfl0z0veanp\nzXs2E01EsW2buB0nnojj8/g43ns8AH/+4s9E4pFkm53cT5ecLpzSObmyw/NbnicUCxFPxFPblBaU\ncub3zgRgwcYFBGNBookoMTtGPBFngG8AVx6XPHn09jdvpzHaSCwRS26TiHFql1O57aTbALjw/y6k\nPlpPJB5JbTPumHHcd3ryAikDfjmAmnANNvtOHZjUaxKLxiSX2ztnxTkEYs2X07ug1wUsHpNcTm/2\nX2fTGG1s1j6xdGIqeK/+YjWReAS35U7dvhyUjy06FhubLCuLLEcWbsvdbBT33tPuxTRM3JYbj8ND\nlpVFl5wuqfa1l63FZbqSbY6s/VbGWHnhoVcVmXXyrEO27x29lvbHMODaawNccUWAiROPzHKBIiLS\nuih4f8WxTxxLbbj5qibnl57PkrHJ5exOe/406iJ1zdrP63keT457EoDRy0dTH60/aPuEFRP2a//y\n/q/44xWHfPxNb9x0wPa9wfuut+46YH17g/f/fPA/BKIBLNPCYSQv6BGOh1PB+yP/R4RiodTFPhym\nI7VcHCSXbPPGvTgtJ07TicN0pEarAab0n0LcjuMwHbgt934rTSwYtQADA5flwmW6cFpOOno6ptpX\nXbQKy7TIsrJwWa7UPvZad/k6DuWx0Y8dsv2KY684ZPve0WuRI+Wtt1x8/rnF5ZcHOessBW4RkfZM\nq5p8xR8+/wPVdc1HvHsW9GRM9zEAPPf354jGo8kGIzlHt0d+D87odgYAy7csJ5aIYRgGRnIDjsk7\nhmFdhgHw8raXidtxTMNM3jDpktuFQR0GAbD2X2uxbRvDMDAxsUyL4qxi+hT1AeCDyg+wsTENE8uw\nsAyLPFcenXM6A7CrcVfy96aFaZipcPzlk+RaK60EkJnULwf3u995+OEPC+nVK8arr+7B2cILvKhv\nMpP6JTNpVRNpCQreX6EPxMylvslM6pf9JRLw4IN5PPZYHqedFubnP/dTWNjyH7Xqm8ykfslMCt7S\nEjTVRETkCIrH4aabivjjHz1MntzIvHm1uFzprkpERDKBgreIyBFkWdC9e5wf/7iWadMatXKJiIik\nKHiLiBwBH3zgwLYNBg6M8uMf1x3+ASIi0u4cpQsVi4i0HytXZjFpUjFz5hTQus6aERGRlqTgLSLy\nLdk2PPJILlOnejn++BhLl/o1tURERA5KU01ERL6FYNBg1qxCXnrJw8UXB3jggRqystJdlYiIZDIF\nbxGRb8E0bXbvNpk7t5bp03USpYiIHJ6Ct4jIN7Bpk5MePWIUFdksX16FQ5+iIiLyNWmOt4jI1/T8\n8x4uuqiYefPyARS6RUTkG9GfDRGRw4hG4ac/zeepp3IZMSLMf/6nlgsUEZFvTsFbROQQ/H6TadOK\nKC93M3VqA3Pn1mmkW0REvhX9+RAROYTGRoPPPnOwYEE1l14aTHc5IiLSiil4i4gcQHm5i6FDI3Tv\nHuevf92tpQJFROQ708mVIiJfEovBf/93PpdeWszy5R4AhW4RETkiNOItItKkqspk+vTkfO4pUxq4\n6CJNLRERkSNHwVtEBHjvPSc33FCE329pPreIiBwVCt4iIkBFhYllwYoVlZxwQjTd5YiISBukOd4i\n0m4Fg7BmjRuAcePCrFlTodAtIiJHjYK3iLRLn39uMWlSMdde62X7dgvQSZQiInJ0KXiLSLuzerWb\nc87pwOefO/jFL/x06xZPd0kiItIOKHiLSLth2/Dgg3lcc42PLl3ivPLKHsaPD6e7LBERaScUvEWk\n3TAMiEbhiisaeemlPfTooZFuERFpOVrVRETavLffdmFZNkOGRJk9ux5TQw4iIpIG+vMjIm1WIgGL\nF+dy2WU+HnggH0ChW0RE0kYj3iLSJvn9JrfdVsjq1Vmcd16Qhx6qSXdJIiLSzil4i0ib849/WFxy\nSTFVVSbz5tVw7bUBDCPdVYmISHun4C0ibU5JSZzhw8NMndrIwIG6II6IiGQGzXYUkTahstJk5sxC\nqqpMnE547LEahW4REckoCt4i0uqtXeti/PgO/N//edi0yZnuckRERA5IwVtEWq1oFO67L4/Jk30U\nFCR4+eU9jBunC+KIiEhmUvAWkVbr7rvzWbgwjyuvDLByZSX9+sXSXZKIiMhB6eRKEWl1wmFwu2Ha\ntAbKyiJMnBhKd0kiIiKHpeAtIq1GQ4PB3LkFVFaa/PKXfkpKEpSUKHSLiEjroKkmItIqbNrk5Kyz\nOvD733s48cQotp3uikRERL4ZjXiLSEaLx2HRolzmz8+jU6c4v/99FaecEkl3WSIiIt+YRrxFJKP5\n/SY//3kOEyaE+NOf9ih0i4hIq6URbxHJOLYNb7zhZsyYMB06JHjttT107ZrQZd9FRKRV04i3iGQU\nv99g2rQirr3Wx4oVHgBKShS6RUSk9dOIt4hkjDVr3NxxR/Ky73Pm1DFpUjDdJYmIiBwxCt4ikhEe\neCCPRx/No2/fKM88U8WAAboYjoiItC2aaiIiGWHw4AjTpzfwyit7FLpFRKRN0oi3iKRFOAwPPZRH\nbq7Nbbc1MHZsmLFjw+kuS0RE5KjRiLeItLjNm52cfXYHFi/OY/duSxfDERGRdkEj3iLSYsJhePjh\nPJ54IpcOHRI8+2wVY8ZolFtERNoHjXiLSIt57z0nixblcsklQVavrlDoFhGRdkUj3iJyVIVCsG6d\nm7FjwwwZEmX16j307auTJ0VEpP3RiLeIHDUbNjg566wOXHedl3/8wwJQ6BYRkXZLwVtEjrhAwOAn\nP8ln0qRiAgGDX/7ST48e8XSXJSIiklZfa6rJ5s2bWbZsGYlEgrFjxzJp0qRm7dFolIULF7Jt2zby\n8vKYOXMmHTt2BGDFihWsXr0a0zS5/vrrGTRoEAA/+MEPyMrKwjRNLMvi/vvvP8IvTUTSIRyGs87q\nwLZtDq69tpE5c+rIzdWyJSIiIocN3olEgqVLlzJ37lx8Ph933nknZWVldOvWLbXN6tWrycnJ4fHH\nH2fdunU899xzzJo1i+3bt1NeXs7DDz9MdXU1d999N48++iimmRxo/8lPfkJ+fv7Re3Ui0mKCQQOP\nx8bthuuua6RfvyjDhkXSXZaIiEjGOOxUk61bt9K5c2c6deqEw+Fg+PDhrF+/vtk2GzZsYNSoUQAM\nHTqUDz74ANu2Wb9+PcOHD8fpdNKxY0c6d+7M1q1bj8oLEZH0sG343//NYtiwjqxd6wLghhsaFbpF\nRES+4rAj3n6/H5/Pl7rv8/n49NNPD7qNZVlkZ2dTX1+P3++nT58+qe28Xi9+vz91f968eQCMHz+e\ncePGHfD5V61axapVqwC4//77KS4u/rqv7VtxOBxH/Tnk21HfZJ7PP4epU52sXOmlrCxB374FFBdr\nWkmm0DGTmdQvmUn9Ii0hbcsJ3n333Xi9Xmpra7nnnnvo2rUr/fr122+7cePGNQvllZWVR7Wu4uLi\no/4c8u2obzLL009nM29ecqrYXXfVMmVKI5YF6qLMoWMmM6lfMtPR6JeuXbse0f1J63fYqSZer5eq\nqqrU/aqqKrxe70G3icfjBAIB8vLy9nus3+9PPXbvz4KCAoYMGaIpKCKtTEODybBhETZtijJ1ajJ0\ni4iIyMEdNnj36tWLnTt3UlFRQSwWo7y8nLKysmbbDB48mDVr1gDw9ttv079/fwzDoKysjPLycqLR\nKBUVFezcuZPevXsTCoUIBoMAhEIh3nvvPY455pgj/+pE5IipqzOYM6eAP/4xC4AZMxp45hk/PXqk\nty4REZHW4rBTTSzLYsqUKcybN49EIsHo0aPp3r07zz//PL169aKsrIwxY8awcOFCbrnlFnJzc5k5\ncyYA3bt3Z9iwYdx+++2YpskNN9yAaZrU1tby0EMPAckR8tNPPz21zKCIZBbbhhUrPNx9dz6VlSYd\nOsQ591wwdRUAERGRb8SwbbtVnQm1Y8eOo7p/zb3LXOqblrdli4M5cwp46y03gwZFuPfeWk48Mdps\nG/VL5lLfZCb1S2bSHG9pCWk7uVJEMt8777j4+GMn999fw+TJAc3jFhER+Q4UvEUkxbbhpZeyiMcN\nLrwwyOTJASZMCOH1JtJdmoiISKunWZoiAsDf/+7g0kt93HSTl9/+NhvbTs7jVugWERE5MjTiLdLO\n1dYazJ+fx9NP55CXZ3PffTVcdVUAw0h3ZSIiIm2LgrdIO7dunZtly3K4+uoAP/pRHV5vqzrfWkRE\npNVQ8BZph8rLXXzxhcXllwc555wQf/5zBb17x9NdloiISJumOd4i7cg//2kxdWoRl15azJIlucTj\nYBgodIuIiLQAjXiLtAN1dQaPP57L0qW5mKbNj35Ux7RpDVoeUEREpAUpeIu0Ax984OSJJ3K5+OIg\ns2fX0aWLVioRERFpaQreIm2QbcOf/uRm2zYH06c3Mnx4hL/8pYKePTWlREREJF00x1ukjdm0ycml\nl/q4/nofy5dnE226wrtCt4iISHopeIu0EV98YTFtWhHnndeBLVsc3HNPDa++ugenM92ViYiICGiq\niUibUVdn8Oc/u5k5s57p0xvIy9N63CIiIplEwVuklaqrM1iyJJeqKpOf/ayW/v1jvPvubgVuERGR\nDKWpJiKtTDAITzyRw7BhnXj00Tzq6w3iTdO3FbpFREQyl0a8RVqRdetc3HprEbt2WYweHeI//qOe\ngQOj6S5LREREvgYFb5EMF41Cba1JcXGCkpI4PXvGWLSomqFDI+kuTURERL4BTTURyVDxOPz+9x5G\njerI7bcXAtCjR5zf/a5KoVtERKQV0oi3SIaJx+F//9fDo4/msnWrk379olx9dWO6yxIREZHvSMFb\nJMM8/nguDz6Yz3HHRXnyST8TJoQw9d2UiIhIq6fgLZJm0Sj84Q8eSkvjDBkS4corA/TpE+OccxS4\nRURE2hL9WRdJk1AInn46m9NP78jttxexfLkHgE6dEpx7rkK3iIhIW6MRb5E0ePbZbB5+OI+KCovB\ngyPMm1fF2LHhdJclIiIiR5GCt0gL8ftNCgsTmCbs3m3Rt2+MhQurGT48gmGkuzoRERE52hS8RY6y\nzz+3ePLJXH77Ww8LF9ZwzjkhZs2qx7LSXZmIiIi0JAVvkaPkvfecPPlkDi+95ME04eKLAxx7bPIq\nkwrdIiIi7Y+Ct8hREI3Ctdd6CQQMbryxkalTG+jSJZHuskRERCSNFLxFjoBg0OB3v/OwcmUWzz7r\nx+mEpUv99O4dIz/fTnd5IiIikgEUvEW+g3/9y+Lpp7P59a9zqKkxOeGECLt3W5SUxDn55Gi6yxMR\nEZEMouAt8i29846LSy7xYdtw9tkhbryxkVNO0QolIiIicmAK3iJfUzBosGKFB8uyufzyICedFOG2\n2xq4/PIA3brF012eiIiIZDgFb5HD2LbN4plncli+PJvaWpMxY0JcfnkQpxPuuKM+3eWJiIhIK6Hg\nLXII996bx6JFeTidNhMmBLnuugBDhkTSXZaIiIi0QgreIl/yz39a/PrX2UyZ0kinTglOPTVCdnYd\nkycH6NhRywGKiIjIt6fgLe1eKASvvZbFb36Tw1/+4sY0bfr3jzJxYoixY8OMHRtOd4kiIiLSBih4\nS7tWXW1w+umdqKkx6dYtxg9/WMcVVwR0sRsRERE54hS8pV2prDR58UUPe/aY3HlnPUVFNtdf38gp\np4Q5/fQIppnuCkVERKStUvCWNi8chlWrsli+PJs//9lNLGZQVhYhHq/HsuCHP9TKJCIiInL0KXhL\nm5RomilimvDQQ3ksXpxH585xvv/9Bi65JMixx8bSW6CIiIi0Owre0mbYNnz4oYMVK7J58UUPjz1W\nzWmnRbjyygCnnRZhxIgwlpXuKkVERKS9UvCWVq+21uDJJ3N56SUP27Y5cDhsRo0K4/HYAJSWxikt\n1ZUlRUREJL0UvKXVsW345BMHNTUmQ4dGcLlsli3LYeDAKNOmNTBhQgivV6uSiIiISGZR8JZWwbbh\n3XcNfv3rPFauTI5sH398lFWr9uDxwIYNu8nJsdNdpoiIiMhBKXhLxorHSc3JvvnmQl580YllORg+\nPMLUqQ2cdVYota1Ct4iIiGQ6BW/JKLW1BmvWuHn99SzefNPN2rV78HoTTJoU5NxzXQwdugevVyFb\nREREWh8Fb8kIGzY4ue++fNavdxGPG/h8cc48M0wwaAAwfnyY4uIElZUK3SIiItI6KXhLi6utNfjr\nX928+aab884LMXJkcgWS+nqTGTMaGDs2zMknR7T0n4iIiLQpCt7SIoJBgyeeyOHNN7PYtMlJPG6Q\nm5ugX78oI0dC//4xXn99T7rLFBERETlqFLzliEsk4KOPHPz1r248Hptrrw3gctk89VQOPXrEufnm\nBkaNCnPSSRGcznRXKyIiItIyFLzliHnuuWxWrXLzzjtuampMAMaMCXHttQEsC9avr0hd1EZERESk\nvVHwlm+socFg0yYnGza42L7dYv78WgDeeMPNli1OzjknyNChEU47LUyXLvsuZKPQLSIiIu2Zgrcc\nUiIBZnLwmuef9/DUUzl89JGTRMLAMGyOOy5GMGjg8dg88UQ1bnd66xURERHJVArekmLbsH27xfvv\nO3nvPSebN7v429+crF5dQZcuCcJhg8JCm1tvbWDIkAgnnRShoGDfKLZCt4iIiMjBKXi3U9EobNvm\n4KOPnAwfHqZTpwS/+U02P/pRIQCWZXP88VEmTgwSjyfX0v63fwvwb/8WSGfZIiIiIq2Wgncbl0hA\nLAYuF2zdarFwYR4ff+xgyxYnkUgyUC9e7OeCC0IMHx7m3ntrOOGEKMcfHyUrK83Fi4iIiLQhCt5t\nSEODwRtvuNm2zcG2bQ62bnXw6acO7rqrjquvDhCPG6xd6+b446OMGNFIv37JgN2rVwyAHj3i9Oih\nEW0RERGRo0HBuxWJRGDjRhdffGHx+ecO/vGP5M8LLwxw3XUBamsNZszwYhg2JSVxSktjXHVVgGOP\njQJw7LExNm7cneZXISIiItI+fa3gvXnzZpYtW0YikWDs2LFMmjSpWXs0GmXhwoVs27aNvLw8Zs6c\nSceOHQFYsWIFq1evxjRNrr/+egYNGvS19tle2HZyvrXLlZwS8vLLHnbtMtm922LXLosdOyzG76xK\nsQAACeJJREFUjw9x880NRCIGF19cDIBh2HTtGueYY+JkZydPcOzSJcGqVRX06BHD40nnqxIRERGR\nrzps8E4kEixdupS5c+fi8/m48847KSsro1u3bqltVq9eTU5ODo8//jjr1q3jueeeY9asWWzfvp3y\n8nIefvhhqqurufvuu3n00UcBDrvP1sS2wUhOl+azzywaGkzq6w0aGgzq6kw6d45z+ukRbBtuvbUQ\nv9+kqsqkstKiqsrkkksCPPhgLaYJt91WSCxm4PEk6Nw5QefOcfLykmth5+ba/Pa3lXTtGqdbt/h+\nq4iYJhx/fKyFX72IiIiIfB2HDd5bt26lc+fOdOrUCYDhw4ezfv36ZiF5w4YNXHrppQAMHTqUp556\nCtu2Wb9+PcOHD8fpdNKxY0c6d+7M1q1bAQ67z3S57DIHfr+PRCJ5YqJtw/DhEe64ox6ACROKqakx\nCYcNQiGDYNDgnHOCLFpUA8CZZ3YgEDCb7XPixCCnnx7BMOCTT5w4HDYdOiTo1y9GcXGcwYOTU0FM\nE1avrqBDhwR5eXYqzH/ZiBGRo/sGiIiIiMhRcdjg7ff78fl8qfs+n49PP/30oNtYlkV2djb19fX4\n/X769OmT2s7r9eL3+1P7OdQ+91q1ahWrVq0C4P7776e4uPjrvrZvpbHRIBZzYprgcCTDcE6ORXFx\ncnh50CCLSCQ5NSQ72yY72+akk1ypupYujeN0xsnPh/x8yMuz6dDBoqAg2b5x4951r82mmwNwA7kA\nHOWX16o5HI6j3v/yzalfMpf6JjOpXzKT+kVaQsafXDlu3DjGjRuXul9ZWXlUn++Pfyw+4HPs/dW9\n9x74cXvbR47cvy0a3dcu315x8YH7RtJL/ZK51DeZSf2SmY5Gv3Tt2vWI7k9aP/NwG3i9XqqqqlL3\nq6qq8Hq9B90mHo8TCATIy8vb77F+vx+v1/u19ikiIiIi0pYcNnj36tWLnTt3UlFRQSwWo7y8nLKy\nsmbbDB48mDVr1gDw9ttv079/fwzDoKysjPLycqLRKBUVFezcuZPevXt/rX2KiIiIiLQlh51qYlkW\nU6ZMYd68eSQSCUaPHk337t15/vnn6dWrF2VlZYwZM4aFCxdyyy23kJuby8yZMwHo3r07w4YN4/bb\nb8c0TW644QZMM5n1D7RPEREREZG2yrBt2z78Zpljx44dR3X/mnuXudQ3mUn9krnUN5lJ/ZKZNMdb\nWsJhp5qIiIiIiMh3p+AtIiIiItICFLxFRERERFqAgreIiIiISAtQ8BYRERERaQEK3iIiIiIiLUDB\nW0RERESkBSh4i4iIiIi0AAVvEREREZEWoOAtIiIiItICFLxFRERERFqAgreIiIiISAtQ8BYRERER\naQEK3iIiIiIiLcCwbdtOdxEiIiIiIm2dRry/Yvbs2ekuQQ5CfZOZ1C+ZS32TmdQvmUn9Ii1BwVtE\nREREpAUoeIuIiIiItADrrrvuuivdRWSa0tLSdJcgB6G+yUzql8ylvslM6pfMpH6Ro00nV4qIiIiI\ntABNNRERERERaQEK3iIiIiIiLcCR7gIyyebNm1m2bBmJRIKxY8cyadKkdJfULlVWVrJo0SJqamow\nDINx48YxYcIEGhoaeOSRR9izZw8dOnRg1qxZ5ObmprvcdieRSDB79my8Xi+zZ8+moqKCBQsWUF9f\nT2lpKbfccgsOhz5aWlpjYyNLlizhiy++wDAMbrrpJrp27apjJs1efvllVq9ejWEYdO/enRkzZlBT\nU6NjJg0WL17Mxo0bKSgoYP78+QAH/bti2zbLli1j06ZNuN1uZsyYofnfckRoxLtJIpFg6dKlzJkz\nh0ceeYR169axffv2dJfVLlmWxTXXXMMjjzzCvHnzeO2119i+fTsvvvgiAwcO5LHHHmPgwIG8+OKL\n6S61XVq5ciUlJSWp+7/61a8499xzefzxx8nJyWH16tVprK79WrZsGYMGDWLBggU8+OCDlJSU6JhJ\nM7/fzyuvvML999/P/PnzSSQSlJeX65hJk1GjRjFnzpxmvzvYMbJp0yZ27drFY489xve//31+8Ytf\npKNkaYMUvJts3bqVzp0706lTJxwOB8OHD2f9+vXpLqtdKioqSo0seDweSkpK8Pv9rF+/njPOOAOA\nM844Q/2TBlVVVWzcuJGxY8cCYNs2H374IUOHDgWSf9jULy0vEAjw8ccfM2bMGAAcDgc5OTk6ZjJA\nIpEgEokQj8eJRCIUFhbqmEmTfv367feNz8GOkQ0bNjBy5EgMw6Bv3740NjZSXV3d4jVL26Pvtpr4\n/X58Pl/qvs/n49NPP01jRQJQUVHBZ599Ru/evamtraWoqAiAwsJCamtr01xd+/P0009z9dVXEwwG\nAaivryc7OxvLsgDwer34/f50ltguVVRUkJ+fz+LFi/nnP/9JaWkp1113nY6ZNPN6vZx//vncdNNN\nuFwuTjzxREpLS3XMZJCDHSN+v5/i4uLUdj6fD7/fn9pW5NvSiLdkrFAoxPz587nuuuvIzs5u1mYY\nBoZhpKmy9undd9+loKBA8xwzUDwe57PPPuPMM8/kgQcewO127zetRMdMy2toaGD9+vUsWrSIJ598\nklAoxObNm9NdlhyEjhFpCRrxbuL1eqmqqkrdr6qqwuv1prGi9i0WizF//nxGjBjBqaeeCkBBQQHV\n1dUUFRVRXV1Nfn5+mqtsXz755BM2bNjApk2biEQiBINBnn76aQKBAPF4HMuy8Pv9Om7SwOfz4fP5\n6NOnDwBDhw7lxRdf1DGTZu+//z4dO3ZMve+nnnoqn3zyiY6ZDHKwY8Tr9VJZWZnaTplAjhSNeDfp\n1asXO3fupKKiglgsRnl5OWVlZekuq12ybZslS5ZQUlLCeeedl/p9WVkZb775JgBvvvkmQ4YMSVeJ\n7dLkyZNZsmQJixYtYubMmQwYMIBbb72V/v378/bbbwOwZs0aHTdpUFhYiM/nY8eOHUAy8HXr1k3H\nTJoVFxfz6aefEg6HsW071S86ZjLHwY6RsrIy1q5di23bbNmyhezsbE0zkSNCV678ko0bN/LMM8+Q\nSCQYPXo0F110UbpLapf+/ve/81//9V8cc8wxqa/9rrzySvr06cMjjzxCZWWllkZLsw8//JCXXnqJ\n2bNns3v3bhYsWEBDQwM9e/bklltuwel0prvEducf//gHS5YsIRaL0bFjR2bMmIFt2zpm0uyFF16g\nvLwcy7Lo0aMH06dPx+/365hJgwULFvDRRx9RX19PQUEBl112GUOGDDngMWLbNkuXLuVvf/sbLpeL\nGTNm0KtXr3S/BGkDFLxFRERERFqAppqIiIiIiLQABW8RERERkRag4C0iIiIi0gIUvEVEREREWoCC\nt4iIiIhIC1DwFhERERFpAQreIiIiIiIt4P8D3GhcNr4D4FIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2374281048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "plt.style.use('ggplot')\n",
    "plt.rcParams['figure.figsize'] = 10, 8\n",
    "def _sample(logits, t):\n",
    "    res = logits / t\n",
    "    res = np.exp(res) / np.sum(np.exp(res))\n",
    "    return res\n",
    "\n",
    "N = 100\n",
    "x = np.arange(N)\n",
    "before = np.array([1.0+i**2 for i in range(N)])\n",
    "before /= before.sum()\n",
    "\n",
    "plt.plot(x, before, 'b--', label='before')\n",
    "\n",
    "after = _sample(before, 0.1)\n",
    "plt.plot(x, after, 'g--', label='temp=0.01')\n",
    "\n",
    "after = _sample(before, 0.2)\n",
    "print(after.argmax())\n",
    "plt.plot(x, after, 'r--', label='temp=0.001')\n",
    "\n",
    "\n",
    "plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " plot(*args, **kwargs)\n",
      "\n",
      "Plot lines and/or markers to the\n",
      ":class:`~matplotlib.axes.Axes`.  *args* is a variable length\n",
      "argument, allowing for multiple *x*, *y* pairs with an\n",
      "optional format string.  For example, each of the following is\n",
      "legal::\n",
      "\n",
      "    plot(x, y)        # plot x and y using default line style and color\n",
      "    plot(x, y, 'bo')  # plot x and y using blue circle markers\n",
      "    plot(y)           # plot y using x as index array 0..N-1\n",
      "    plot(y, 'r+')     # ditto, but with red plusses\n",
      "\n",
      "If *x* and/or *y* is 2-dimensional, then the corresponding columns\n",
      "will be plotted.\n",
      "\n",
      "If used with labeled data, make sure that the color spec is not\n",
      "included as an element in data, as otherwise the last case\n",
      "``plot(\"v\",\"r\", data={\"v\":..., \"r\":...)``\n",
      "can be interpreted as the first case which would do ``plot(v, r)``\n",
      "using the default line style and color.\n",
      "\n",
      "If not used with labeled data (i.e., without a data argument),\n",
      "an arbitrary number of *x*, *y*, *fmt* groups can be specified, as in::\n",
      "\n",
      "    a.plot(x1, y1, 'g^', x2, y2, 'g-')\n",
      "\n",
      "Return value is a list of lines that were added.\n",
      "\n",
      "By default, each line is assigned a different style specified by a\n",
      "'style cycle'.  To change this behavior, you can edit the\n",
      "axes.prop_cycle rcParam.\n",
      "\n",
      "The following format string characters are accepted to control\n",
      "the line style or marker:\n",
      "\n",
      "================    ===============================\n",
      "character           description\n",
      "================    ===============================\n",
      "``'-'``             solid line style\n",
      "``'--'``            dashed line style\n",
      "``'-.'``            dash-dot line style\n",
      "``':'``             dotted line style\n",
      "``'.'``             point marker\n",
      "``','``             pixel marker\n",
      "``'o'``             circle marker\n",
      "``'v'``             triangle_down marker\n",
      "``'^'``             triangle_up marker\n",
      "``'<'``             triangle_left marker\n",
      "``'>'``             triangle_right marker\n",
      "``'1'``             tri_down marker\n",
      "``'2'``             tri_up marker\n",
      "``'3'``             tri_left marker\n",
      "``'4'``             tri_right marker\n",
      "``'s'``             square marker\n",
      "``'p'``             pentagon marker\n",
      "``'*'``             star marker\n",
      "``'h'``             hexagon1 marker\n",
      "``'H'``             hexagon2 marker\n",
      "``'+'``             plus marker\n",
      "``'x'``             x marker\n",
      "``'D'``             diamond marker\n",
      "``'d'``             thin_diamond marker\n",
      "``'|'``             vline marker\n",
      "``'_'``             hline marker\n",
      "================    ===============================\n",
      "\n",
      "\n",
      "The following color abbreviations are supported:\n",
      "\n",
      "==========  ========\n",
      "character   color\n",
      "==========  ========\n",
      "'b'         blue\n",
      "'g'         green\n",
      "'r'         red\n",
      "'c'         cyan\n",
      "'m'         magenta\n",
      "'y'         yellow\n",
      "'k'         black\n",
      "'w'         white\n",
      "==========  ========\n",
      "\n",
      "In addition, you can specify colors in many weird and\n",
      "wonderful ways, including full names (``'green'``), hex\n",
      "strings (``'#008000'``), RGB or RGBA tuples (``(0,1,0,1)``) or\n",
      "grayscale intensities as a string (``'0.8'``).  Of these, the\n",
      "string specifications can be used in place of a ``fmt`` group,\n",
      "but the tuple forms can be used only as ``kwargs``.\n",
      "\n",
      "Line styles and colors are combined in a single format string, as in\n",
      "``'bo'`` for blue circles.\n",
      "\n",
      "The *kwargs* can be used to set line properties (any property that has\n",
      "a ``set_*`` method).  You can use this to set a line label (for auto\n",
      "legends), linewidth, anitialising, marker face color, etc.  Here is an\n",
      "example::\n",
      "\n",
      "    plot([1,2,3], [1,2,3], 'go-', label='line 1', linewidth=2)\n",
      "    plot([1,2,3], [1,4,9], 'rs',  label='line 2')\n",
      "    axis([0, 4, 0, 10])\n",
      "    legend()\n",
      "\n",
      "If you make multiple lines with one plot command, the kwargs\n",
      "apply to all those lines, e.g.::\n",
      "\n",
      "    plot(x1, y1, x2, y2, antialiased=False)\n",
      "\n",
      "Neither line will be antialiased.\n",
      "\n",
      "You do not need to use format strings, which are just\n",
      "abbreviations.  All of the line properties can be controlled\n",
      "by keyword arguments.  For example, you can set the color,\n",
      "marker, linestyle, and markercolor with::\n",
      "\n",
      "    plot(x, y, color='green', linestyle='dashed', marker='o',\n",
      "         markerfacecolor='blue', markersize=12).\n",
      "\n",
      "See :class:`~matplotlib.lines.Line2D` for details.\n",
      "\n",
      "The kwargs are :class:`~matplotlib.lines.Line2D` properties:\n",
      "\n",
      "  agg_filter: unknown\n",
      "  alpha: float (0.0 transparent through 1.0 opaque) \n",
      "  animated: [True | False] \n",
      "  antialiased or aa: [True | False] \n",
      "  axes: an :class:`~matplotlib.axes.Axes` instance \n",
      "  clip_box: a :class:`matplotlib.transforms.Bbox` instance \n",
      "  clip_on: [True | False] \n",
      "  clip_path: [ (:class:`~matplotlib.path.Path`, :class:`~matplotlib.transforms.Transform`) | :class:`~matplotlib.patches.Patch` | None ] \n",
      "  color or c: any matplotlib color \n",
      "  contains: a callable function \n",
      "  dash_capstyle: ['butt' | 'round' | 'projecting'] \n",
      "  dash_joinstyle: ['miter' | 'round' | 'bevel'] \n",
      "  dashes: sequence of on/off ink in points \n",
      "  drawstyle: ['default' | 'steps' | 'steps-pre' | 'steps-mid' | 'steps-post'] \n",
      "  figure: a :class:`matplotlib.figure.Figure` instance \n",
      "  fillstyle: ['full' | 'left' | 'right' | 'bottom' | 'top' | 'none'] \n",
      "  gid: an id string \n",
      "  label: string or anything printable with '%s' conversion. \n",
      "  linestyle or ls: ['solid' | 'dashed', 'dashdot', 'dotted' | (offset, on-off-dash-seq) | ``'-'`` | ``'--'`` | ``'-.'`` | ``':'`` | ``'None'`` | ``' '`` | ``''``]\n",
      "  linewidth or lw: float value in points \n",
      "  marker: :mod:`A valid marker style <matplotlib.markers>`\n",
      "  markeredgecolor or mec: any matplotlib color \n",
      "  markeredgewidth or mew: float value in points \n",
      "  markerfacecolor or mfc: any matplotlib color \n",
      "  markerfacecoloralt or mfcalt: any matplotlib color \n",
      "  markersize or ms: float \n",
      "  markevery: [None | int | length-2 tuple of int | slice | list/array of int | float | length-2 tuple of float]\n",
      "  path_effects: unknown\n",
      "  picker: float distance in points or callable pick function ``fn(artist, event)`` \n",
      "  pickradius: float distance in points \n",
      "  rasterized: [True | False | None] \n",
      "  sketch_params: unknown\n",
      "  snap: unknown\n",
      "  solid_capstyle: ['butt' | 'round' |  'projecting'] \n",
      "  solid_joinstyle: ['miter' | 'round' | 'bevel'] \n",
      "  transform: a :class:`matplotlib.transforms.Transform` instance \n",
      "  url: a url string \n",
      "  visible: [True | False] \n",
      "  xdata: 1D array \n",
      "  ydata: 1D array \n",
      "  zorder: any number \n",
      "\n",
      "kwargs *scalex* and *scaley*, if defined, are passed on to\n",
      ":meth:`~matplotlib.axes.Axes.autoscale_view` to determine\n",
      "whether the *x* and *y* axes are autoscaled; the default is\n",
      "*True*.\n",
      "\n",
      ".. note::\n",
      "    In addition to the above described arguments, this function can take a\n",
      "    **data** keyword argument. If such a **data** argument is given, the\n",
      "    following arguments are replaced by **data[<arg>]**:\n",
      "\n",
      "    * All arguments with the following names: 'x', 'y'.\n"
     ]
    }
   ],
   "source": [
    "np.info(plt.plot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
